{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Converting DICOM files to PNG\n\nWe will convert the DICOM images to PNG, so it is way faster to load and train on it.","metadata":{}},{"cell_type":"markdown","source":"This is based on other kaggle notebooks with reference inside the functions.\n\nIt need the metadata from DICOM images to work, where I provided in a separated dataset:\n\nhttps://www.kaggle.com/datasets/ibombonato/unifesp-xray-body-part-dicom-metadata-csv\n\n**Please upvote de notebook and the dataset if it helps you :D**","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm.auto import tqdm\nfrom PIL import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-11T14:44:43.522667Z","iopub.execute_input":"2022-04-11T14:44:43.523178Z","iopub.status.idle":"2022-04-11T14:44:43.633673Z","shell.execute_reply.started":"2022-04-11T14:44:43.523094Z","shell.execute_reply":"2022-04-11T14:44:43.632970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_metadata = pd.read_csv(\"../input/unifesp-xray-body-part-dicom-metadata-csv/dicom_metadata_train.csv\")\ndf_metadata_test = pd.read_csv(\"../input/unifesp-xray-body-part-dicom-metadata-csv/dicom_metadata_test.csv\")\n\ndf = pd.read_csv(\"../input/unifesp-x-ray-body-part-classifier/train.csv\")\ndf_test = pd.read_csv(\"../input/unifesp-x-ray-body-part-classifier/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:44:46.701541Z","iopub.execute_input":"2022-04-11T14:44:46.701795Z","iopub.status.idle":"2022-04-11T14:44:46.818957Z","shell.execute_reply.started":"2022-04-11T14:44:46.701764Z","shell.execute_reply":"2022-04-11T14:44:46.818238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_df = pd.merge(df, df_metadata)\ndicom_df_test = pd.merge(df_test, df_metadata_test)\nprint(dicom_df.shape)\ndicom_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:44:48.886405Z","iopub.execute_input":"2022-04-11T14:44:48.886683Z","iopub.status.idle":"2022-04-11T14:44:48.960462Z","shell.execute_reply.started":"2022-04-11T14:44:48.886649Z","shell.execute_reply":"2022-04-11T14:44:48.959101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n\n# Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:44:52.137096Z","iopub.execute_input":"2022-04-11T14:44:52.137998Z","iopub.status.idle":"2022-04-11T14:44:52.341126Z","shell.execute_reply.started":"2022-04-11T14:44:52.137963Z","shell.execute_reply":"2022-04-11T14:44:52.340390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:51:24.486284Z","iopub.execute_input":"2022-04-11T14:51:24.486692Z","iopub.status.idle":"2022-04-11T14:51:24.492726Z","shell.execute_reply.started":"2022-04-11T14:51:24.486640Z","shell.execute_reply":"2022-04-11T14:51:24.492100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n\ndef convert_image(image_path, split):\n    xray = read_xray(image_path)\n    im = resize(xray, size=512, keep_ratio=True)  \n    new_path = f'./images/{split}/{Path(image_path).name.replace(\".dcm\", \".png\")}'\n    #new_path = f'./{Path(image_path).name.replace(\".dcm\", \".png\")}'\n    im.save(new_path)\n       \n    return new_path","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:50:45.458219Z","iopub.execute_input":"2022-04-11T14:50:45.458507Z","iopub.status.idle":"2022-04-11T14:50:45.464505Z","shell.execute_reply.started":"2022-04-11T14:50:45.458477Z","shell.execute_reply":"2022-04-11T14:50:45.463488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = dicom_df.iloc[10]['fname']\nxray = read_xray(image_path)\nimg_xray = Image.fromarray(xray)\nimg_sample = resize(xray, size=512, keep_ratio=True)  \nimg_sample.save('./image_png.png')","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:56:35.542236Z","iopub.execute_input":"2022-04-11T14:56:35.542528Z","iopub.status.idle":"2022-04-11T14:56:41.690692Z","shell.execute_reply.started":"2022-04-11T14:56:35.542499Z","shell.execute_reply":"2022-04-11T14:56:41.688739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_png = plt.imread('./image_png.png')\n\nfig = plt.figure(figsize=(12,10))\n\nplt.subplot(2,2,1)\nplt.imshow(img_xray, cmap='gray')\nplt.title('File DICOM')\n\nplt.subplot(2,2,2)\nplt.imshow(file_png, cmap='gray')\nplt.title('File PNG')","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:56:44.722367Z","iopub.execute_input":"2022-04-11T14:56:44.723001Z","iopub.status.idle":"2022-04-11T14:56:46.826568Z","shell.execute_reply.started":"2022-04-11T14:56:44.722965Z","shell.execute_reply":"2022-04-11T14:56:46.825234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = [x for x in Path('../input/unifesp-x-ray-body-part-classifier/train').rglob('*') if x.is_file()]\ntest_files = [x for x in Path('../input/unifesp-x-ray-body-part-classifier/test').rglob('*') if x.is_file()]","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:48:28.814025Z","iopub.execute_input":"2022-04-11T14:48:28.814366Z","iopub.status.idle":"2022-04-11T14:49:03.135116Z","shell.execute_reply.started":"2022-04-11T14:48:28.814343Z","shell.execute_reply":"2022-04-11T14:49:03.134040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./images\n!mkdir ./images/train\n!mkdir ./images/test","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:49:03.136669Z","iopub.execute_input":"2022-04-11T14:49:03.136878Z","iopub.status.idle":"2022-04-11T14:49:03.953918Z","shell.execute_reply.started":"2022-04-11T14:49:03.136851Z","shell.execute_reply":"2022-04-11T14:49:03.952674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import Parallel, delayed\nr_train = Parallel(n_jobs=2)(delayed(convert_image)(x, 'train') for x in train_files)\nr_test = Parallel(n_jobs=2)(delayed(convert_image)(x, 'test') for x in test_files)","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:51:29.032725Z","iopub.execute_input":"2022-04-11T14:51:29.033113Z","iopub.status.idle":"2022-04-11T14:51:51.877597Z","shell.execute_reply.started":"2022-04-11T14:51:29.033085Z","shell.execute_reply":"2022-04-11T14:51:51.876807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df(files, results, df):\n    df_paths = pd.DataFrame({'fname': files, 'image_path': results})\n    df_paths['fname'] = df_paths['fname'].astype(str)\n    df_result = pd.merge(df, df_paths)\n    return df_result","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:52:55.771800Z","iopub.execute_input":"2022-04-11T14:52:55.772076Z","iopub.status.idle":"2022-04-11T14:52:55.777858Z","shell.execute_reply.started":"2022-04-11T14:52:55.772047Z","shell.execute_reply":"2022-04-11T14:52:55.777155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dicom_paths_train = get_df(train_files, r_train, dicom_df)","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:52:05.463838Z","iopub.execute_input":"2022-04-11T14:52:05.464053Z","iopub.status.idle":"2022-04-11T14:52:05.477644Z","shell.execute_reply.started":"2022-04-11T14:52:05.464030Z","shell.execute_reply":"2022-04-11T14:52:05.477129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dicom_paths_test = get_df(test_files, r_test, dicom_df_test)","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:52:09.115698Z","iopub.execute_input":"2022-04-11T14:52:09.116116Z","iopub.status.idle":"2022-04-11T14:52:09.126147Z","shell.execute_reply.started":"2022-04-11T14:52:09.116092Z","shell.execute_reply":"2022-04-11T14:52:09.125612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dicom_paths_train.shape, df_dicom_paths_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:52:10.839686Z","iopub.execute_input":"2022-04-11T14:52:10.840018Z","iopub.status.idle":"2022-04-11T14:52:10.844778Z","shell.execute_reply.started":"2022-04-11T14:52:10.839994Z","shell.execute_reply":"2022-04-11T14:52:10.844355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dicom_paths_train.to_csv('train_df.csv', index=False)\ndf_dicom_paths_test.to_csv('test_df.csv', index=False)\ndf_dicom_paths_test[['SOPInstanceUID', 'Target']].to_csv('sample_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-11T14:52:13.076770Z","iopub.execute_input":"2022-04-11T14:52:13.077173Z","iopub.status.idle":"2022-04-11T14:52:13.090740Z","shell.execute_reply.started":"2022-04-11T14:52:13.077149Z","shell.execute_reply":"2022-04-11T14:52:13.089753Z"},"trusted":true},"execution_count":null,"outputs":[]}]}