{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Full Dataset - https://www.kaggle.com/rashmibanthia/osic-pulmonary-jpg \n\n\nThis kernel only has a sample, but I used this locally to create the full dataset. Hopefully it will be easier to work with on Colab. Please let me know if I missed something. \n\n\nQuite a bit of code is from - \nhttps://www.kaggle.com/aakashnain/dicom-to-jpg-png-on-steroids\n\n\n-----------\n\nVer 2: Fixed files for patient ID00011637202177653955184 and ID00052637202186188008618/4.dcm \n\n\nVer 1: \n(All files for patient - ID00011637202177653955184 seems corrupt, \nAlso this file - ID00052637202186188008618/4.dcm) \n","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\n# import cv2\nfrom PIL import Image\nimport glob\nimport time\nimport gdcm\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom pathlib import Path, PosixPath\n\nimport dask as dd\nimport dask.array as da","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get list of all training DCM files","execution_count":null},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Path to the data\ndata_dir = Path('../input/osic-pulmonary-fibrosis-progression/train')\n\n# get the list of all the dcm files recursively\ntrain_all_files = list(data_dir.glob(\"**/*.dcm\"))\n\ntrain_all_dirs = list(data_dir.glob(\"**\"))[1:] #Excluding this directory - ../input/osic-pulmonary-fibrosis-progression/train\n\nprint(\"Number of train dcm files found: \", len(train_all_files), \" in \", len(train_all_dirs), \" directories.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One patient per directory - We have 176 patients in training data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the path to output directory\noutdir = \"./processed_images/train\"\n\n# Make the directory\nif not os.path.exists(outdir):\n    os.makedirs(outdir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Convert DICOM to JPG/PNG  \ndef convert_images(filename,outdir,img_type='jpg'):\n    \"\"\"Reads a dcm file and saves the files as png/jpg\n    \n    Args:\n        filename: path to the dcm file\n        img_type: format of the processed file (jpg or png)\n        \n    \"\"\"\n    \n    # extract the name of the file\n    name = filename.parts[-1]\n    \n    # read the dcm file\n    try:\n        ds = pydicom.read_file(str(filename)) \n        img = ds.pixel_array\n\n        outdir = outdir + \"/\" + str(filename).split(\"/\")[4] #PatientID\n        # Make the directory\n        if not os.path.exists(outdir):\n            os.makedirs(outdir)\n\n        #https://stackoverflow.com/questions/56956198/pil-converting-from-i16-to-jpeg-produce-white-image\n        norm = (img.astype(np.float)-img.min())*255.0 / (img.max()-img.min())\n        # save the image as jpg/png\n        if img_type==\"jpg\":\n            Image.fromarray(norm.astype(np.uint8)).save(outdir + \"/\" + name.replace('.dcm','.jpg'))\n        else:\n            Image.fromarray(norm.astype(np.uint8)).save(outdir + \"/\" + name.replace('.dcm','.png'))\n            \n    except:\n        print(filename)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_all_files))\n\ntrain_all_files = train_all_files[0:10] #sample only 10 files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # First using the simple way: the for loop\n# t = time.time()\n# for f in train_all_files:\n#     convert_images(f,outdir)\n# print(\"Time taken : \", time.time() - t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using dask and Running only for 10 samples\nall_images = [dd.delayed(convert_images)(train_all_files[x],outdir) for x in range(len(train_all_files))]\n\nt = time.time()\ndd.compute(all_images)\nprint(\"Time taken when using all cores: \", time.time()-t)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check to ensure every directory has the same set of files as input","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Path to the data\ndata_dir = Path('./processed_images/train')\n\n# get the list of all the dcm files recursively\ntrain_all_files_output = list(data_dir.glob(\"**/*.jpg\"))\n\ntrain_all_dirs_output = list(data_dir.glob(\"**\"))[1:] #Excluding this directory - ../input/osic-pulmonary-fibrosis-progression/train\n\nprint(\"Number of train dcm files found: \", len(train_all_files_output), \" in \", len(train_all_dirs_output), \" directories.\")\n\ntrain_all_files_output = [str(i) for  i in train_all_files_output]\ntrain_all_files_output = [i.split(\"/\")[1] + \"/\" + i.split(\"/\")[2]+\"/\" + i.split(\"/\")[3][:-4]  for  i in train_all_files_output]\ntrain_all_files_output = sorted(train_all_files_output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_all_files_tmp = [str(i) for  i in train_all_files]\ntrain_all_files_tmp =  [i.split(\"/\")[3] + \"/\" + i.split(\"/\")[4]+\"/\" + i.split(\"/\")[5][:-4]  for  i in train_all_files_tmp]\ntrain_all_files_tmp = sorted(train_all_files_tmp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(train_all_files_tmp)):\n    if train_all_files_tmp[i]!=train_all_files_output[i]:\n        print(train_all_files_output[i],train_all_files_tmp[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize Dicom and Visualize JPG\n\n(Sanity check)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_all_files_tmp[1],train_all_files_output[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pyd = pydicom.read_file(\"../input/osic-pulmonary-fibrosis-progression/\" + train_all_files_tmp[1]+\".dcm\")\n# pyd = pydicom.read_file('../input/osic-pulmonary-fibrosis-progression/train/ID00052637202186188008618/4.dcm')\nimage_data = pyd.pixel_array\nplt.imshow(image_data, cmap=plt.cm.bone);\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pyd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(Image.open('./processed_images/' + train_all_files_output[1] + '.jpg'));","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Process Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Path to the data\ndata_dir = Path('../input/osic-pulmonary-fibrosis-progression/test')\n\n# get the list of all the dcm files recursively\ntest_all_files = list(data_dir.glob(\"**/*.dcm\"))\n\ntest_all_dirs = list(data_dir.glob(\"**\"))[1:] #Excluding this directory - ../input/osic-pulmonary-fibrosis-progression/test\n\nprint(\"Number of test dcm files found: \", len(test_all_files), \" in \", len(test_all_dirs), \" directories.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"One patient per directory - We have only 5 patients in test set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the path to output directory\noutdir = \"./processed_images/test\"\n\n# Make the directory\nif not os.path.exists(outdir):\n    os.makedirs(outdir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Using dask \nall_images = [dd.delayed(convert_images)(test_all_files[x],outdir) for x in range(len(test_all_files[0:10]))]\n\nt = time.time()\ndd.compute(all_images)\nprint(\"Time taken when using all cores: \", time.time()-t)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = Path('./processed_images/test')\n\n\n# get the list of all the dcm files recursively\ntest_all_files_output = list(data_dir.glob(\"**/*.jpg\"))\nlen(test_all_files_output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -R ./processed_images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}