{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Analysis of the image files for the [SIIM COVID-19 Detection](https://www.kaggle.com/c/siim-covid19-detection/overview) competition**.\n\n**Conclusions**:\n- The image must be preprocessed as explained at [this link](https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way).\n- There are several group of images we can observed based on the pixel values.\n- It is better to use matplotlib instead of plotly as long as interactivity is not needed.","metadata":{}},{"cell_type":"markdown","source":"# CONFS","metadata":{}},{"cell_type":"code","source":"ROOT = '/kaggle/input/siim-covid19-detection'\nEXAMPLE = '/kaggle/input/siim-covid19-detection/train/cd5dd5e6f3f5/b2ee36aa2df5/d8ba599611e5.dcm'","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:25:41.050368Z","iopub.execute_input":"2021-06-27T19:25:41.050916Z","iopub.status.idle":"2021-06-27T19:25:41.057211Z","shell.execute_reply.started":"2021-06-27T19:25:41.050802Z","shell.execute_reply":"2021-06-27T19:25:41.055627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IMPORTS","metadata":{}},{"cell_type":"code","source":"# required to handle compress pixel values\n!conda install --yes --channel=conda-forge gdcm","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:25:41.063993Z","iopub.execute_input":"2021-06-27T19:25:41.064459Z","iopub.status.idle":"2021-06-27T19:26:16.24282Z","shell.execute_reply.started":"2021-06-27T19:25:41.064419Z","shell.execute_reply":"2021-06-27T19:26:16.24159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport matplotlib.pyplot as plt","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","tags":[],"execution":{"iopub.status.busy":"2021-06-27T19:47:11.511085Z","iopub.execute_input":"2021-06-27T19:47:11.511554Z","iopub.status.idle":"2021-06-27T19:47:11.518103Z","shell.execute_reply.started":"2021-06-27T19:47:11.511518Z","shell.execute_reply":"2021-06-27T19:47:11.516387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","tags":[],"execution":{"iopub.status.busy":"2021-06-27T19:26:52.056712Z","iopub.execute_input":"2021-06-27T19:26:52.05705Z","iopub.status.idle":"2021-06-27T19:26:52.067928Z","shell.execute_reply.started":"2021-06-27T19:26:52.057019Z","shell.execute_reply":"2021-06-27T19:26:52.066985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATASETS","metadata":{}},{"cell_type":"code","source":"def compute_statistics(file: pathlib.Path) -> pd.Series:\n    \"\"\"Generate statistics from the pixels of a dicom file.\"\"\"\n    pixels = pydicom.read_file(file).pixel_array # get pixels\n    stats = pd.Series(pixels.flatten()).describe()\n    stats['rows'] = pixels.shape[0]\n    stats['cols'] = pixels.shape[1]\n    stats.name = file.stem\n    return stats","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ThreadPoolExecutor(100) as executor:\n    files = pathlib.Path(ROOT).glob('**/*.dcm')\n    stats = executor.map(compute_statistics, files)\ndf = pd.DataFrame(stats)\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ANALYSIS","metadata":{}},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- The number of pixels on the image is not evenly distributed.\n- The relation between mean and standard deviation shows clusters.\n- The same conclusion can be said about the median and interquartile range.\n- On the other hand, the number of rows and columns follows a linear relationship.","metadata":{}},{"cell_type":"code","source":"px.histogram(df, x='count')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.scatter(df, x='mean', y='std')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.scatter(df, x='50%', y=df['75%']-df['25%'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.scatter(df, x='rows', y='cols')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EXAMPLES","metadata":{}},{"cell_type":"code","source":"def read_pixels(path: pathlib.Path, voi_lut: bool = True, fix_monochrome: bool = True):\n    \"\"\"Read a dicom file and convert its pixel values to a proper image format/range.\"\"\"\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        pixels = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        pixels = dicom.pixel_array\n    # depending on the photometric interpretation, X-ray may look inverted. Fix the problem by reversing values\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        pixels = np.amax(pixels) - pixels\n    # normalize the pixel values to a range between 0 and 1: (X - min) / (max - min)\n    pixels = (pixels - np.min(pixels)) / (np.max(pixels) - np.min(pixels))\n    # convert the range from [0, 1] to [0, 255] and cast to uint8\n    pixels = (pixels * 255).astype(np.uint8)\n    return pixels","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:26:52.069553Z","iopub.execute_input":"2021-06-27T19:26:52.069981Z","iopub.status.idle":"2021-06-27T19:26:52.081294Z","shell.execute_reply.started":"2021-06-27T19:26:52.069925Z","shell.execute_reply":"2021-06-27T19:26:52.080219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Color are inverted on some images (monochrome1).\n- The goal is to have black lungs on a white background.\n- The fix monochrome attribute helps to fix this problem.\n- the VOI LUT attribute seems to make images a little ligther.","metadata":{}},{"cell_type":"code","source":"title = f'VOI_LUT=False, FIX_MONOCHROME=False'\npixels = read_pixels(EXAMPLE, False, False)\nplt.figure(figsize=(12,12))\nplt.imshow(pixels, 'gray')\nplt.title(title);","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:47:46.412049Z","iopub.execute_input":"2021-06-27T19:47:46.412846Z","iopub.status.idle":"2021-06-27T19:47:47.260665Z","shell.execute_reply.started":"2021-06-27T19:47:46.412788Z","shell.execute_reply":"2021-06-27T19:47:47.259460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = f'VOI_LUT=True, FIX_MONOCHROME=False'\npixels = read_pixels(EXAMPLE, True, False)\nplt.figure(figsize=(12,12))\nplt.imshow(pixels, 'gray')\nplt.title(title);","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:48:05.842106Z","iopub.execute_input":"2021-06-27T19:48:05.842525Z","iopub.status.idle":"2021-06-27T19:48:06.679537Z","shell.execute_reply.started":"2021-06-27T19:48:05.842491Z","shell.execute_reply":"2021-06-27T19:48:06.678690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = f'VOI_LUT=False, FIX_MONOCHROME=True'\npixels = read_pixels(EXAMPLE, False, True)\nplt.figure(figsize=(12,12))\nplt.imshow(pixels, 'gray')\nplt.title(title);","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:48:06.681118Z","iopub.execute_input":"2021-06-27T19:48:06.681610Z","iopub.status.idle":"2021-06-27T19:48:07.554558Z","shell.execute_reply.started":"2021-06-27T19:48:06.681559Z","shell.execute_reply":"2021-06-27T19:48:07.553124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title = f'VOI_LUT=True, FIX_MONOCHROME=True'\npixels = read_pixels(EXAMPLE, True, True)\nplt.figure(figsize=(12,12))\nplt.imshow(pixels, 'gray')\nplt.title(title);","metadata":{"execution":{"iopub.status.busy":"2021-06-27T19:48:07.556758Z","iopub.execute_input":"2021-06-27T19:48:07.557422Z","iopub.status.idle":"2021-06-27T19:48:08.405808Z","shell.execute_reply.started":"2021-06-27T19:48:07.557369Z","shell.execute_reply":"2021-06-27T19:48:08.404493Z"},"trusted":true},"execution_count":null,"outputs":[]}]}