{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA-MICCAI Brain Tumor Radiogenomics Classification","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport pydicom\nfrom tqdm import tqdm\nfrom multiprocessing import Pool","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-02T09:11:26.023447Z","iopub.execute_input":"2021-09-02T09:11:26.023798Z","iopub.status.idle":"2021-09-02T09:11:26.346874Z","shell.execute_reply.started":"2021-09-02T09:11:26.023699Z","shell.execute_reply":"2021-09-02T09:11:26.345975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract Metadata from DICOM\n\nThanks to ([this code](https://www.kaggle.com/c/rsna-miccai-brain-tumor-radiogenomic-classification/discussion/252942)) by @pestipeti and @echo9k","metadata":{}},{"cell_type":"code","source":"FIELDS = [\n    'AccessionNumber',\n    'AcquisitionMatrix',\n    #'B1rms',  # Empty\n    #'BitsAllocated',  # = 16\n    #'BitsStored',  # = 16\n    'Columns',\n    'ConversionType',\n    #'DiffusionBValue',  # 0 or empty\n    #'DiffusionGradientOrientation',  # [0.0, 0.0, 0.0] or empty\n    'EchoNumbers',\n    #'EchoTime',  # empty\n    'EchoTrainLength',\n    'FlipAngle',\n    #'HighBit',  # = 15\n    #'HighRRValue',  #  0 or empty\n    'ImageDimensions',  # 2 or epty\n    'ImageFormat',\n    'ImageGeometryType',\n    'ImageLocation',\n    'ImageOrientation',\n    'ImageOrientationPatient',\n    'ImagePosition',\n    'ImagePositionPatient',\n    #'ImageType',  # ['DERIVED', 'SECONDARY']\n    'ImagedNucleus',\n    'ImagingFrequency',\n    'InPlanePhaseEncodingDirection',\n    'InStackPositionNumber',\n    'InstanceNumber',\n    #'InversionTime',   # empty\n    #'Laterality',  # empty\n    #'LowRRValue',  # empty\n    'MRAcquisitionType',\n    'MagneticFieldStrength',\n    #'Modality',  # MR\n    'NumberOfAverages',\n    'NumberOfPhaseEncodingSteps',\n    'PatientID',\n    'PatientName',\n    #'PatientPosition',  # HFS\n    'PercentPhaseFieldOfView',\n    'PercentSampling',\n    #'PhotometricInterpretation',  # MONOCHROME2\n    'PixelBandwidth',\n    #'PixelPaddingValue',  # empty or 0\n    'PixelRepresentation',\n    'PixelSpacing',\n    #'PlanarConfiguration',  # 0 or empty\n    #'PositionReferenceIndicator',  # 'NA' or empty\n    'PresentationLUTShape',\n    'ReconstructionDiameter',\n    #'RescaleIntercept',  # = 0\n    #'RescaleSlope',  # = 1\n    #'RescaleType',  # = US\n    'Rows',\n    'SAR',\n    'SOPClassUID',\n    'SOPInstanceUID',\n    #'SamplesPerPixel',  # = 1\n    'SeriesDescription',\n    'SeriesInstanceUID',\n    'SeriesNumber',\n    'SliceLocation',\n    'SliceThickness',\n    'SpacingBetweenSlices',\n    'SpatialResolution',\n    'SpecificCharacterSet',\n    'StudyInstanceUID',\n    #'TemporalResolution',  # 0 or empty\n    #'TransferSyntaxUID',  # = 1.2.840.10008.1.2\n    #'TriggerWindow',  # = 0\n    'WindowCenter',\n    'WindowWidth'\n]\n\n# All of the FM fields are empty\nFM_FIELDS = [\n    'FileMetaInformationGroupLength',\n    'FileMetaInformationVersion',\n    'ImplementationClassUID',\n    'ImplementationVersionName',\n    'MediaStorageSOPClassUID',\n    'MediaStorageSOPInstanceUID',\n    'SourceApplicationEntityTitle',\n    'TransferSyntaxUID',\n]\n\n\ndef get_meta_info(dicom):\n    row = {f: dicom.get(f) for f in FIELDS}\n    row_fm = {f: dicom.file_meta.get(f) for f in FM_FIELDS}\n    row_other = {\n        #'is_original_encoding': dicom.is_original_encoding,  # = True\n        #'is_implicit_VR': dicom.is_implicit_VR,  # = True\n        #'is_little_endian': dicom.is_little_endian, # = True\n        'timestamp': dicom.timestamp,\n    }\n    return {**row,\n            #**row_fm,  # All are emtpy\n            **row_other}\n\n\ndef get_dicom_files(input_dir, ds='train'):\n    dicoms = []\n    for subdir, dirs, files in os.walk(f\"{input_dir}/{ds}\"):\n        for filename in files:\n            filepath = subdir + os.sep + filename\n            if filepath.endswith(\".dcm\"):\n                dicoms.append(filepath)\n    return dicoms\n\n\ndef process_dicom(dicom_src, _x):\n    dicom = pydicom.dcmread(dicom_src)\n    file_data = dicom_src.split(\"/\")\n    file_src = \"/\".join(file_data[-4:])\n    tmp = {\"BraTS21ID\": file_data[-3], \"dataset\": file_data[-4], \"type\": file_data[-2], \"dicom_src\": f\"./{file_src}\"}\n    tmp.update(get_meta_info(dicom))\n    return tmp\n\n\ndef update(res):\n    if res is not None:\n        final.append(res)\n    pbar.update()\n\n\ndef error(e):\n    print(e)\n","metadata":{"execution":{"iopub.status.busy":"2021-09-02T09:11:26.350583Z","iopub.execute_input":"2021-09-02T09:11:26.350852Z","iopub.status.idle":"2021-09-02T09:21:00.607206Z","shell.execute_reply.started":"2021-09-02T09:11:26.350826Z","shell.execute_reply":"2021-09-02T09:21:00.606341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args={}\nargs['input'] = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\nargs['output'] = './'\nargs['dataset'] = 'train'\nargs['n_jobs'] = 20\nargs['debug'] = 0    \n\nfinal = []\n\nif __name__ == \"__main__\":\n    dicom_files = get_dicom_files(args[\"input\"], args[\"dataset\"])\n\n    if args[\"debug\"]:\n        dicom_files = dicom_files[:1000]\n\n    pool = Pool(processes=args[\"n_jobs\"])\n    pbar = tqdm(total=len(dicom_files))\n\n    for dicom_file in dicom_files:\n        pool.apply_async(\n            process_dicom,\n            args=(dicom_file, ''),\n            callback=update,\n            error_callback=error,\n        )\n\n    pool.close()\n    pool.join()\n    pbar.close()\n\n    train_final = pd.DataFrame(final)\n    train_final.to_csv(f\"{args['output']}/dicom_meta_{args['dataset']}.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final","metadata":{"execution":{"iopub.status.busy":"2021-09-02T09:21:00.609455Z","iopub.execute_input":"2021-09-02T09:21:00.609892Z","iopub.status.idle":"2021-09-02T09:21:01.907925Z","shell.execute_reply.started":"2021-09-02T09:21:00.609840Z","shell.execute_reply":"2021-09-02T09:21:01.906999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Do the same for Test Data","metadata":{}},{"cell_type":"code","source":"args={}\nargs['input'] = '../input/rsna-miccai-brain-tumor-radiogenomic-classification'\nargs['output'] = './'\nargs['dataset'] = 'test'\nargs['n_jobs'] = 20\nargs['debug'] = 0\n\nfinal = []\n\nif __name__ == \"__main__\":\n    dicom_files = get_dicom_files(args[\"input\"], args[\"dataset\"])\n\n    if args[\"debug\"]:\n        dicom_files = dicom_files[:1000]\n\n    pool = Pool(processes=args[\"n_jobs\"])\n    pbar = tqdm(total=len(dicom_files))\n\n    for dicom_file in dicom_files:\n        pool.apply_async(\n            process_dicom,\n            args=(dicom_file, ''),\n            callback=update,\n            error_callback=error,\n        )\n\n    pool.close()\n    pool.join()\n    pbar.close()\n\n    test_final = pd.DataFrame(final)\n    test_final.to_csv(f\"{args['output']}/dicom_meta_{args['dataset']}.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-02T09:21:18.489511Z","iopub.execute_input":"2021-09-02T09:21:18.489842Z","iopub.status.idle":"2021-09-02T09:22:50.400672Z","shell.execute_reply.started":"2021-09-02T09:21:18.489811Z","shell.execute_reply":"2021-09-02T09:22:50.399440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final","metadata":{"execution":{"iopub.status.busy":"2021-09-02T09:22:50.402274Z","iopub.execute_input":"2021-09-02T09:22:50.402570Z","iopub.status.idle":"2021-09-02T09:22:50.629447Z","shell.execute_reply.started":"2021-09-02T09:22:50.402540Z","shell.execute_reply":"2021-09-02T09:22:50.628726Z"},"trusted":true},"execution_count":null,"outputs":[]}]}