{"cells":[{"metadata":{},"cell_type":"markdown","source":"This code read all train dicom files and transform every feature into column in Dataframe **except images** and then merge it with train csv.\n\nIt can be reproduced with test data.\n\nAlso it separate arrays features like 'ImageType' or 'PixelSpacing' into new columns.\n\nThen it save result as pkl file that i'm using it for the rest of my work.\n\nIt can be usfull for your work then i share it.\n"},{"metadata":{},"cell_type":"markdown","source":"# Train Part"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom\nfrom pydicom import dcmread\nfrom pydicom.data import get_testdata_files\nimport glob, os\nfrom collections import defaultdict\nimport tqdm\nimport gc\nimport seaborn as sns\nimport ast","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"col_name = ['ImageType', 'SOPInstanceUID', 'Modality', 'Manufacturer','ManufacturerModelName', 'PatientName', 'PatientID', 'PatientSex', 'DeidentificationMethod',\n'BodyPartExamined', 'SliceThickness', 'KVP', 'SpacingBetweenSlices', 'GantryDetectorTilt', 'TableHeight', 'RotationDirection', 'XRayTubeCurrent', 'ConvolutionKernel',                 \n'PatientPosition', 'StudyInstanceUID', 'SeriesInstanceUID', 'StudyID', 'InstanceNumber', 'ImagePositionPatient', 'ImageOrientationPatient', 'FrameOfReferenceUID',          \n'PositionReferenceIndicator', 'SliceLocation', 'SamplesPerPixel', 'PhotometricInterpretation', 'Rows', 'Columns', 'PixelSpacing','BitsAllocated','BitsStored',                    \n'HighBit', 'PixelRepresentation','WindowCenter','WindowWidth','RescaleIntercept','RescaleSlope']\n\ndf = pd.DataFrame(columns=col_name)\nmy_dict = defaultdict(list)\n\nfor name in tqdm.tqdm(glob.glob('/kaggle/input/osic-pulmonary-fibrosis-progression/train/*/*')):\n    ds = pydicom.read_file(name)\n    for i in col_name :\n        if i in ds :\n            my_dict[i].append(str(ds[i].value))\n        else:\n            my_dict[i].append(np.nan)\n    df = pd.concat([df, pd.DataFrame(my_dict)], ignore_index = True)\n    del my_dict\n    my_dict = defaultdict(list)\ngc.collect()\n\nfor name in tqdm.tqdm(glob.glob('/kaggle/input/osic-pulmonary-fibrosis-progression/test/*/*')):\n    ds = pydicom.read_file(name)\n    for i in col_name :\n        if i in ds :\n            my_dict[i].append(str(ds[i].value))\n        else:\n            my_dict[i].append(np.nan)\n    df = pd.concat([df, pd.DataFrame(my_dict)], ignore_index = True)\n    del my_dict\n    my_dict = defaultdict(list)\ngc.collect()\n#     ds = pydicom.read_file(name)\n#     df = pd.DataFrame(ds.values())\n#     df[0] = df[0].apply(lambda x: pydicom.dataelem.DataElement_from_raw(x) if isinstance(x, pydicom.dataelem.RawDataElement) else x)\n#     df['name'] = df[0].apply(lambda x: x.name)\n#     df['value'] = df[0].apply(lambda x: x.value)\n#     df = df[['name', 'value']]\n\n#     df = df.set_index('name').T.reset_index(drop=True)\n#     df.drop('Pixel Data', axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv'), pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')], ignore_index = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\nData = df.merge(train , left_on='PatientID', right_on='Patient')\n\ntype_dict_all = ['ORIGINAL', 'PRIMARY', 'AXIAL', 'CT_SOM5 SPI', 'HELIX', 'CT_SOM5 SEQ', 'SECONDARY', 'DERIVED', 'JP2K LOSSY 6:1', 'VOLUME', 'OTHER', 'CSA MPR', 'CSAPARALLEL', \n                'CSA RESAMPLED', 'REFORMATTED', 'AVERAGE', 'CT_SOM7 SPI DUAL', 'STD', 'SNRG', 'DET_AB']\nfor i in type_dict_all:\n    Data[i] = np.nan\nfor index, row in tqdm.tqdm(Data.iterrows()):\n    for i in type_dict_all:\n        if i in Data.loc[index, 'ImageType'] :\n            Data.loc[index, i] = 1\n            \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Data.fillna(0, inplace = True)\n\nData.loc[Data['ImagePositionPatient'] == 0, 'ImagePositionPatient'] = '[0,0,0]'\nData['ImagePositionPatient'] = Data['ImagePositionPatient'].apply(ast.literal_eval)\nData[['ImagePositionPatient_x','ImagePositionPatient_y', 'ImagePositionPatient_z']] = pd.DataFrame(Data.ImagePositionPatient.tolist(), index= Data.index)\n\nData.loc[Data['ImageOrientationPatient'] == 0, 'ImageOrientationPatient'] = '[0,0,0,0,0,0]'\nData['ImageOrientationPatient'] = Data['ImageOrientationPatient'].apply(ast.literal_eval)\nData[['ImageOrientationPatient_a','ImageOrientationPatient_b', 'ImageOrientationPatient_c', 'ImageOrientationPatient_d', 'ImageOrientationPatient_e', 'ImageOrientationPatient_f']] = pd.DataFrame(Data.ImageOrientationPatient.tolist(), index= Data.index)\n\ntmp1 = Data[['ImagePositionPatient_x','ImagePositionPatient_y', 'ImagePositionPatient_z', 'ImageOrientationPatient_a','ImageOrientationPatient_b', 'ImageOrientationPatient_c']]\ntmp1.columns = ['x','y','z','a','b','c']\n\ntmp1['Cos'] = 'red'\ntmp2 = Data[['ImagePositionPatient_x','ImagePositionPatient_y', 'ImagePositionPatient_z', 'ImageOrientationPatient_d','ImageOrientationPatient_e', 'ImageOrientationPatient_f']]\ntmp2.columns = ['x','y','z','a','b','c']\ntmp2['Cos'] = 'blue'\n\ncos = pd.concat([tmp1, tmp2], ignore_index = True)\ncos['width'] = 10\n\ncos[['a','b','c']] = cos[['a','b','c']] * 200\n\nData.loc[Data['PixelSpacing'] == 0, 'PixelSpacing'] = '[0,0]'\nData['PixelSpacing'] = Data['PixelSpacing'].apply(ast.literal_eval)\nData[['PixelSpacing_row','PixelSpacing_column']] = pd.DataFrame(Data.PixelSpacing.tolist(), index= Data.index)\n\nData.to_pickle('output_data.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Data.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Part"},{"metadata":{"trusted":true},"cell_type":"code","source":"col_name = ['ImageType', 'SOPInstanceUID', 'Modality', 'Manufacturer','ManufacturerModelName', 'PatientName', 'PatientID', 'PatientSex', 'DeidentificationMethod',\n'BodyPartExamined', 'SliceThickness', 'KVP', 'SpacingBetweenSlices', 'GantryDetectorTilt', 'TableHeight', 'RotationDirection', 'XRayTubeCurrent', 'ConvolutionKernel',                 \n'PatientPosition', 'StudyInstanceUID', 'SeriesInstanceUID', 'StudyID', 'InstanceNumber', 'ImagePositionPatient', 'ImageOrientationPatient', 'FrameOfReferenceUID',          \n'PositionReferenceIndicator', 'SliceLocation', 'SamplesPerPixel', 'PhotometricInterpretation', 'Rows', 'Columns', 'PixelSpacing','BitsAllocated','BitsStored',                    \n'HighBit', 'PixelRepresentation','WindowCenter','WindowWidth','RescaleIntercept','RescaleSlope']\n\ndf = pd.DataFrame(columns=col_name)\nmy_dict = defaultdict(list)\n\nfor name in tqdm.tqdm(glob.glob('/kaggle/input/osic-pulmonary-fibrosis-progression/test/*/*.dcm')):\n    ds = pydicom.read_file(name)\n    for i in col_name :\n        if i in ds :\n            my_dict[i].append(str(ds[i].value))\n        else:\n            my_dict[i].append(np.nan)\n    df = pd.concat([df, pd.DataFrame(my_dict)], ignore_index = True)\n    del my_dict\n    my_dict = defaultdict(list)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Patient_Week']]\ntest = sub.merge(test.drop('Weeks', axis=1), on=\"Patient\")\n\nData = df.merge(test , left_on='PatientID', right_on='Patient')\n\ntype_dict_all = ['ORIGINAL', 'PRIMARY', 'AXIAL', 'CT_SOM5 SPI', 'HELIX', 'CT_SOM5 SEQ', 'SECONDARY', 'DERIVED', 'JP2K LOSSY 6:1', 'VOLUME', 'OTHER', 'CSA MPR', 'CSAPARALLEL', \n                'CSA RESAMPLED', 'REFORMATTED', 'AVERAGE', 'CT_SOM7 SPI DUAL', 'STD', 'SNRG', 'DET_AB']\nfor i in type_dict_all:\n    Data[i] = np.nan\nfor index, row in tqdm.tqdm(Data.iterrows()):\n    for i in type_dict_all:\n        if i in Data.loc[index, 'ImageType'] :\n            Data.loc[index, i] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Data.fillna(0, inplace = True)\n\nData.loc[Data['ImagePositionPatient'] == 0, 'ImagePositionPatient'] = '[0,0,0]'\nData['ImagePositionPatient'] = Data['ImagePositionPatient'].apply(ast.literal_eval)\nData[['ImagePositionPatient_x','ImagePositionPatient_y', 'ImagePositionPatient_z']] = pd.DataFrame(Data.ImagePositionPatient.tolist(), index= Data.index)\n\nData.loc[Data['ImageOrientationPatient'] == 0, 'ImageOrientationPatient'] = '[0,0,0,0,0,0]'\nData['ImageOrientationPatient'] = Data['ImageOrientationPatient'].apply(ast.literal_eval)\nData[['ImageOrientationPatient_a','ImageOrientationPatient_b', 'ImageOrientationPatient_c', 'ImageOrientationPatient_d', 'ImageOrientationPatient_e', 'ImageOrientationPatient_f']] = pd.DataFrame(Data.ImageOrientationPatient.tolist(), index= Data.index)\n\ntmp1 = Data[['ImagePositionPatient_x','ImagePositionPatient_y', 'ImagePositionPatient_z', 'ImageOrientationPatient_a','ImageOrientationPatient_b', 'ImageOrientationPatient_c']]\ntmp1.columns = ['x','y','z','a','b','c']\n\ntmp1['Cos'] = 'red'\ntmp2 = Data[['ImagePositionPatient_x','ImagePositionPatient_y', 'ImagePositionPatient_z', 'ImageOrientationPatient_d','ImageOrientationPatient_e', 'ImageOrientationPatient_f']]\ntmp2.columns = ['x','y','z','a','b','c']\ntmp2['Cos'] = 'blue'\n\ncos = pd.concat([tmp1, tmp2], ignore_index = True)\ncos['width'] = 10\n\ncos[['a','b','c']] = cos[['a','b','c']] * 200\n\nData.loc[Data['PixelSpacing'] == 0, 'PixelSpacing'] = '[0,0]'\nData['PixelSpacing'] = Data['PixelSpacing'].apply(ast.literal_eval)\nData[['PixelSpacing_row','PixelSpacing_column']] = pd.DataFrame(Data.PixelSpacing.tolist(), index= Data.index)\n\nData.to_pickle('output_data_test.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"Data.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}