{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Analysis of the image metadata for the [SIIM COVID-19 Detection](https://www.kaggle.com/c/siim-covid19-detection/overview) competition**.\n\n**Conclusions**:\n- Some fields are obfuscated and cannot be used as-is.\n- The following list gives the potential associated with each field:\n    - *ignored*: can be ignored, no potential for other tasks.\n    - *technical*: can be used to improve image reading/parsing.\n    - *detection*: can be integrated for the object detection task.\n    - *classification*: can be integrated for the study classification task.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"fields = {\n    '': {'ignored'},\n    'AccessionNumber': {'ignored'},\n    'BitsAllocated': {'ignored'},\n    'BitsStored': {'ignored'},\n    'BodyPartExamined': {'ignored'},\n    'CodeMeaning': {'ignored'},\n    'CodeValue': {'ignored'},\n    'CodingSchemeDesignator': {'ignored'},\n    'CodingSchemeVersion': {'ignored'},\n    'Columns': {'ignored'},\n    'DeidentificationMethod': {'ignored'},\n    'DeidentificationMethodCodeSequence': {'ignored'},\n    'FileMetaInformationGroupLength': {'ignored'},\n    'FileMetaInformationVersion': {'ignored'},\n    'HighBit': {'ignored'},\n    'ImageType': {'ignored'},\n    'ImagerPixelSpacing': {'ignored'},\n    'ImplementationClassUID': {'ignored'},\n    'ImplementationVersionName': {'ignored'},\n    'InstanceNumber': {'ignored'},\n    'MediaStorageSOPClassUID': {'ignored'},\n    'MediaStorageSOPInstanceUID': {'ignored'},\n    'Modality': {'ignored'},\n    'PatientID': {'ignored'},\n    'PatientName': {'ignored'},\n    'PatientSex': {'classification', 'detection'},\n    'PhotometricInterpretation': {'technical'},\n    'PixelRepresentation': {'ignored'},\n    'Rows': {'ignored'},\n    'SOPClassUID': {'ignored'},\n    'SOPInstanceUID': {'ignored'},\n    'SamplesPerPixel': {'ignored'},\n    'SeriesInstanceUID': {'ignored'},\n    'SeriesNumber': {'ignored'},\n    'SpecificCharacterSet': {'ignored'},\n    'StudyDate': {'ignored'},\n    'StudyID': {'ignored'},\n    'StudyInstanceUID': {'ignored'},\n    'StudyTime': {'ignored'},\n    'TransferSyntaxUID': {'ignored'},\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:26:33.095480Z","iopub.execute_input":"2021-06-23T19:26:33.096272Z","iopub.status.idle":"2021-06-23T19:26:33.107137Z","shell.execute_reply.started":"2021-06-23T19:26:33.096213Z","shell.execute_reply":"2021-06-23T19:26:33.106221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ALLOWED = {'ignored', 'technical', 'detection', 'classification'}\nfor name, tags in fields.items():\n    assert tags.issubset(ALLOWED), name","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:26:35.373469Z","iopub.execute_input":"2021-06-23T19:26:35.374010Z","iopub.status.idle":"2021-06-23T19:26:35.379519Z","shell.execute_reply.started":"2021-06-23T19:26:35.373958Z","shell.execute_reply":"2021-06-23T19:26:35.378609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CONFS","metadata":{}},{"cell_type":"code","source":"ROOT = '/kaggle/input/siim-covid19-detection'","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:09:11.559455Z","iopub.execute_input":"2021-06-23T19:09:11.560130Z","iopub.status.idle":"2021-06-23T19:09:11.570747Z","shell.execute_reply.started":"2021-06-23T19:09:11.559994Z","shell.execute_reply":"2021-06-23T19:09:11.569721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IMPORTS","metadata":{}},{"cell_type":"code","source":"import pathlib\nimport pydicom\nimport itertools\nimport pandas as pd\nimport plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:09:11.572317Z","iopub.execute_input":"2021-06-23T19:09:11.572693Z","iopub.status.idle":"2021-06-23T19:09:13.152982Z","shell.execute_reply.started":"2021-06-23T19:09:11.572633Z","shell.execute_reply":"2021-06-23T19:09:13.151948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATASETS","metadata":{}},{"cell_type":"code","source":"meta = []\nfor file in pathlib.Path(ROOT).glob('**/*.dcm'):\n    print('.', end='') # progress bar\n    # read the file, but ignore pixel values\n    dcm = pydicom.dcmread(file, stop_before_pixels=True)\n    # iterall will magically convert RawElement from DataElement\n    elements = itertools.chain(dcm.iterall(), dcm.file_meta.iterall())\n    for elem in elements:\n        data = {\n            'VM': elem.VM,\n            'VR': elem.VR,\n            'tag': elem.tag,\n            'name': elem.name,\n            'keyword': elem.keyword,\n            # SQ values are redundant with other elements\n            'data': elem.value if elem.VR != 'SQ' else None,\n            'value': elem.repval,\n            'filename': file.name,\n        }\n        meta.append(data)\ndf = pd.DataFrame(meta)\nprint('DONE:', len(df))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:09:13.154628Z","iopub.execute_input":"2021-06-23T19:09:13.154907Z","iopub.status.idle":"2021-06-23T19:12:25.189888Z","shell.execute_reply.started":"2021-06-23T19:09:13.154881Z","shell.execute_reply":"2021-06-23T19:12:25.188668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wide = df.drop_duplicates(subset=['filename', 'keyword'])\nwide = wide.pivot(index='filename', columns='keyword', values='value')\nwide.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:25.191736Z","iopub.execute_input":"2021-06-23T19:12:25.192103Z","iopub.status.idle":"2021-06-23T19:12:25.615665Z","shell.execute_reply.started":"2021-06-23T19:12:25.192070Z","shell.execute_reply":"2021-06-23T19:12:25.614645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ANALYSIS","metadata":{}},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:25.617121Z","iopub.execute_input":"2021-06-23T19:12:25.617570Z","iopub.status.idle":"2021-06-23T19:12:25.868556Z","shell.execute_reply.started":"2021-06-23T19:12:25.617525Z","shell.execute_reply":"2021-06-23T19:12:25.867527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'#Filename', df['filename'].nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:25.869784Z","iopub.execute_input":"2021-06-23T19:12:25.870059Z","iopub.status.idle":"2021-06-23T19:12:25.953490Z","shell.execute_reply.started":"2021-06-23T19:12:25.870033Z","shell.execute_reply":"2021-06-23T19:12:25.952386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'#Tag', df['tag'].nunique(),'#Name', df['name'].nunique(),'#Keyword', df['keyword'].nunique()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:25.954790Z","iopub.execute_input":"2021-06-23T19:12:25.955074Z","iopub.status.idle":"2021-06-23T19:12:26.107188Z","shell.execute_reply.started":"2021-06-23T19:12:25.955047Z","shell.execute_reply":"2021-06-23T19:12:26.106189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## VM = Value Multiplicity","metadata":{}},{"cell_type":"code","source":"vm_counts = df['VM'].value_counts()\nvm_counts","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:26.109753Z","iopub.execute_input":"2021-06-23T19:12:26.110098Z","iopub.status.idle":"2021-06-23T19:12:26.121332Z","shell.execute_reply.started":"2021-06-23T19:12:26.110059Z","shell.execute_reply":"2021-06-23T19:12:26.119982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(vm_counts)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:26.123286Z","iopub.execute_input":"2021-06-23T19:12:26.123778Z","iopub.status.idle":"2021-06-23T19:12:27.280511Z","shell.execute_reply.started":"2021-06-23T19:12:26.123727Z","shell.execute_reply":"2021-06-23T19:12:27.279540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## VR = Value Representation\n\nhttps://pydicom.github.io/pydicom/stable/guides/element_value_types.html","metadata":{}},{"cell_type":"code","source":"vr_counts = df['VR'].value_counts()\nvr_counts","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:27.281772Z","iopub.execute_input":"2021-06-23T19:12:27.282071Z","iopub.status.idle":"2021-06-23T19:12:27.384772Z","shell.execute_reply.started":"2021-06-23T19:12:27.282041Z","shell.execute_reply":"2021-06-23T19:12:27.383847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(vr_counts)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:27.386117Z","iopub.execute_input":"2021-06-23T19:12:27.386438Z","iopub.status.idle":"2021-06-23T19:12:27.463294Z","shell.execute_reply.started":"2021-06-23T19:12:27.386400Z","shell.execute_reply":"2021-06-23T19:12:27.462349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Keywords","metadata":{}},{"cell_type":"code","source":"keyword_counts = df['keyword'].value_counts()\nkeyword_counts","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:27.464684Z","iopub.execute_input":"2021-06-23T19:12:27.464983Z","iopub.status.idle":"2021-06-23T19:12:27.550755Z","shell.execute_reply.started":"2021-06-23T19:12:27.464953Z","shell.execute_reply":"2021-06-23T19:12:27.549796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.bar(keyword_counts)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:27.552265Z","iopub.execute_input":"2021-06-23T19:12:27.552611Z","iopub.status.idle":"2021-06-23T19:12:27.621657Z","shell.execute_reply.started":"2021-06-23T19:12:27.552579Z","shell.execute_reply":"2021-06-23T19:12:27.620561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Associations","metadata":{}},{"cell_type":"code","source":"for name, values in wide.iteritems():\n    fig = px.bar(values.value_counts(dropna=False), title=name)\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T19:12:27.622714Z","iopub.execute_input":"2021-06-23T19:12:27.622977Z","iopub.status.idle":"2021-06-23T19:12:30.925330Z","shell.execute_reply.started":"2021-06-23T19:12:27.622952Z","shell.execute_reply":"2021-06-23T19:12:30.924664Z"},"trusted":true},"execution_count":null,"outputs":[]}]}