{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport re\nimport cv2\nimport glob\nimport time\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport dask as dd\nimport dask.array as da\nfrom dask.distributed import Client, progress\n\nseed = 111\nnp.random.seed(seed)\n%config IPCompleter.use_jedi = False","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:27:32.28779Z","iopub.execute_input":"2021-07-15T15:27:32.288652Z","iopub.status.idle":"2021-07-15T15:27:33.756461Z","shell.execute_reply.started":"2021-07-15T15:27:32.288531Z","shell.execute_reply":"2021-07-15T15:27:33.755057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the datasets","metadata":{}},{"cell_type":"code","source":"data_path = Path(\"../input/rsna-miccai-brain-tumor-radiogenomic-classification/\")\ntrain_data = data_path / \"train\"\ntest_data = data_path / \"test\"","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:28:25.016168Z","iopub.execute_input":"2021-07-15T15:28:25.016882Z","iopub.status.idle":"2021-07-15T15:28:25.022555Z","shell.execute_reply.started":"2021-07-15T15:28:25.016833Z","shell.execute_reply":"2021-07-15T15:28:25.021013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(data_path / \"train_labels.csv\")\nprint(\"Total number of training samples:\", len(train_df))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:28:52.828159Z","iopub.execute_input":"2021-07-15T15:28:52.828569Z","iopub.status.idle":"2021-07-15T15:28:52.868039Z","shell.execute_reply.started":"2021-07-15T15:28:52.828536Z","shell.execute_reply":"2021-07-15T15:28:52.867051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Because each directory in the training images have 5 digits in its name\n# we will make a small change in our df\ntrain_df[\"BraTS21ID\"] = train_df[\"BraTS21ID\"].apply(lambda x: str(x).zfill(5))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:29:18.080375Z","iopub.execute_input":"2021-07-15T15:29:18.081032Z","iopub.status.idle":"2021-07-15T15:29:18.097563Z","shell.execute_reply.started":"2021-07-15T15:29:18.080985Z","shell.execute_reply":"2021-07-15T15:29:18.096151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualizations**","metadata":{}},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\nrandom_idx = np.random.randint(len(train_df))\ncase = train_df[\"BraTS21ID\"][random_idx]\nlabel = train_df[\"MGMT_value\"][random_idx]\nscans = [\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\"]\n\nall_slices = list(map(str, list((train_data / case / scans[0]).glob(\"*.dcm\"))))\n\n# Sort the slices\nall_slices.sort(key=lambda x: int(re.sub('\\D', '', x)))\n\nnrows = len(all_slices) // 5\nncols = 5\nf, ax = plt.subplots(nrows, ncols, figsize=(20, 40))\n\nfor i in range(len(all_slices)):\n    img = dicom2array(all_slices[i])\n    ax[i // ncols, i % ncols].imshow(img, cmap=\"gray\")\n    ax[i // ncols, i % ncols].axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:31:09.514202Z","iopub.execute_input":"2021-07-15T15:31:09.514671Z","iopub.status.idle":"2021-07-15T15:31:17.258654Z","shell.execute_reply.started":"2021-07-15T15:31:09.514608Z","shell.execute_reply":"2021-07-15T15:31:17.257839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process DICOM in parallel","metadata":{}},{"cell_type":"code","source":"@dd.delayed\ndef read_dicom_file(filepath):\n    return pydicom.read_file(filepath)\n\n@dd.delayed\ndef apply_sequence(dicom):\n    return apply_voi_lut(dicom.pixel_array, dicom)\n\n@dd.delayed\ndef fix_monochrome(data):\n    data = np.amax(data) - data\n    return data\n\n@dd.delayed\ndef preprocess_data(data):\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\ndef process_dicom(slice_path):\n    dicom = read_dicom_file(slice_path)\n    data = apply_sequence(dicom)\n    data = fix_monochrome(data)\n    data = preprocess_data(data)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:32:41.549164Z","iopub.execute_input":"2021-07-15T15:32:41.549552Z","iopub.status.idle":"2021-07-15T15:32:41.559633Z","shell.execute_reply.started":"2021-07-15T15:32:41.549518Z","shell.execute_reply":"2021-07-15T15:32:41.557873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client = Client()\nclient","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:32:58.657932Z","iopub.execute_input":"2021-07-15T15:32:58.658361Z","iopub.status.idle":"2021-07-15T15:33:00.327474Z","shell.execute_reply.started":"2021-07-15T15:32:58.658323Z","shell.execute_reply":"2021-07-15T15:33:00.326063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_slice_data = []\nscans = [\"FLAIR\", \"T1w\", \"T1wCE\", \"T2w\"]\ncount = 0\n\nstart_time = time.time()\n\n# I will process only first 10 samples to keep it simple!\nfor i in range(len(train_df.iloc[:10, :])):\n    case = train_df[\"BraTS21ID\"][i]\n    outputs = []\n    \n    for scan in scans:\n        all_slices = list((train_data / case / scan).glob(\"*.dcm\"))\n        all_slices.sort(key=lambda x: int(re.sub('\\D', '', str(x))))\n        outputs.append(process_dicom(slice_path) for slice_path in all_slices)\n    \n    processed_slice_data.append(dd.compute(outputs)[0])\n    \n    \n    for data in processed_slice_data[i:]:\n        count += len(data[0]) + len(data[1]) + len(data[2]) + len(data[3])\n    print(\"Number of DICOM files processed: \", count)\n    \n    \nprint(f\"\\nTime taken to read all slices for 10 samples: {time.time()-start_time:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:38:02.510348Z","iopub.execute_input":"2021-07-15T15:38:02.510707Z","iopub.status.idle":"2021-07-15T15:39:41.276931Z","shell.execute_reply.started":"2021-07-15T15:38:02.510676Z","shell.execute_reply":"2021-07-15T15:39:41.274553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-15T15:39:59.540353Z","iopub.execute_input":"2021-07-15T15:39:59.540821Z","iopub.status.idle":"2021-07-15T15:39:59.970076Z","shell.execute_reply.started":"2021-07-15T15:39:59.540743Z","shell.execute_reply":"2021-07-15T15:39:59.969161Z"},"trusted":true},"execution_count":null,"outputs":[]}]}