{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":20604,"databundleVersionId":1357052,"sourceType":"competition"},{"sourceId":1479371,"sourceType":"datasetVersion","datasetId":868155}],"dockerImageVersionId":30008,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook shows how the [PNG training set](https://www.kaggle.com/qitvision/pulmonary-fibrosis-progression-png-training-set) images were created.\n\nThe dicoms are converted to RGB images using Fast.ai medical imaging library's systematic windowing and normalization. \nThe method is copied from this excellent [Fibrosis EDA (fast.ai)](https://www.kaggle.com/nxrprime/fibrosis-eda-fast-ai) notebook.\n\nFast.ai's medical imaging library is demonstrated [here](https://www.kaggle.com/jhoward/don-t-see-like-a-radiologist-fastai) by Jeremy Howard.\n\n#### Sample from PNG dataset\n\n![Sample](https://storage.googleapis.com/kagglesdsdata/datasets%2F868155%2F1479371%2Ftrain_png%2FID00007637202177411956430%2F13.png?GoogleAccessId=databundle-worker-v2@kaggle-161607.iam.gserviceaccount.com&Expires=1600027110&Signature=uuNMyEqW0loK35%2FfX7Qb09mmgjbmMvyw%2Behfz1OkT%2BUHzHb7MwACGcfmbdK9EHMMrffSm7xfn90r2Zqs%2BfON64dnbD1TpCzH6RTAkpAWhWDxiX1b8nw%2Fej0zOk96rPCtxBJC7apGMur%2FdNqb0%2FMHcL8lF4eNanUWP7qh4hf1QVFRVuRMbxegOZTeecrzkvehpQFcSogsYffMTSNu6DT3o1BYGpm0p6NeQkmimr06uCuzGbM61ruic9X%2Bnk2P%2B2S72tSrQrcn8Hhy6kIf3I8zQiS4Reu3S5bIrD%2FgIJlozNQrn6r0WRnMkbqpvim%2F%2Fo8ThMiwA%2FM3GKhawghqxG3K3Q%3D%3D)","metadata":{}},{"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install fastai==2.0.8","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport pydicom as dcm\nimport gdcm\nfrom fastai.basics           import *\nfrom fastai.medical.imaging  import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DICOM_DIR = '../input/osic-pulmonary-fibrosis-progression/train/'\nPNG_DIR = '../input/pulmonary-fibrosis-progression-png-training-set/train_png/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get dicom paths for patients","metadata":{}},{"cell_type":"code","source":"from collections.abc import Iterable\n\ndef get_sorted_patient_files(directory):\n    \"\"\"\n    Returns all files patients and their files in numerical order.\n\n    Parameters:\n    directory                    (str) : Parent directory containing patients\n\n    Returns:\n    patient_files   (nested file list) : Outer list has patients and inner lists have their file paths in order\n    \"\"\"\n    sorted_files = []\n    all_files = [[os.path.join(directory,d ,fn) for fn in os.listdir(directory + d)] for d in os.listdir(directory)]\n    for patient_files in all_files: \n        patient_numbers = [int(os.path.basename(fn)[:-4]) for fn in patient_files]\n        zipped_list = zip(patient_numbers, patient_files)\n        zipped_list = sorted(zipped_list)\n        tuples = zip(*zipped_list)\n        patient_numbers, patient_files = [list(_tuple) for _tuple in tuples]\n        sorted_files.append(patient_files)\n    return sorted_files","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom_files = get_sorted_patient_files(DICOM_DIR)\n\ndicom_files[0][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize dicom in three different ways:\n\n- Lung window\n- Subdural window\n- Normalized image\n\nCombine these to a single 3-channel image that can be used for CNN training.","metadata":{}},{"cell_type":"code","source":"# This originates from: https://www.kaggle.com/jhoward/cleaning-the-data-for-rapid-prototyping-fastai\ndef fix_pxrepr(dcm):\n    if dcm.PixelRepresentation != 0 or dcm.RescaleIntercept<-100: return\n    x = dcm.pixel_array + 1000\n    px_mode = 4096\n    x[x>=px_mode] = x[x>=px_mode] - px_mode\n    dcm.PixelData = x.tobytes()\n    dcm.RescaleIntercept = -1000\n\ndef dcm_tfm(fn, im_size=512): \n    try:\n        x = dcm.dcmread(fn)\n        fix_pxrepr(x)\n    except Exception as e:\n        print(e)\n    if x.Rows != im_size or x.Columns != im_size: x.zoom_to((im_size,im_size))\n\n    px = x.scaled_px\n    return TensorImage(px.to_3chan(dicom_windows.lungs,dicom_windows.subdural, bins=None))\n\ndef tensor_to_numpy(tensor):\n    img = tensor.cpu().numpy()\n    return img.transpose(1,2,0)\n\ndef open_dicom_normalized(fn, im_size=512):\n    \"\"\"\n    This function returns normalized numpy image representation from a dicom path.\n    Image channels are: lung window, subdural window, normalized total range\n    \"\"\"\n    img = tensor_to_numpy(dcm_tfm(fn, im_size))\n    return (img*255).astype(np.uint8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_index = 10\nslice_index = 10\n\n_, axs = plt.subplots(1,4,figsize=(16,4))\n\nchannels = ['Lung window', 'Subdural window','Normalized image']\nfor i, (ch, ax) in enumerate(zip(channels,axs.ravel())):\n    ax.imshow(open_dicom_normalized(dicom_files[patient_index][slice_index])[:,:,i], cmap='bone')\n    ax.set_title(ch)\n\naxs[-1].imshow(open_dicom_normalized(dicom_files[patient_index][slice_index]))\naxs[-1].set_title('All three combined')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PNG dataset\n\n[Here](https://www.kaggle.com/qitvision/pulmonary-fibrosis-progression-png-training-set) is the training dataset in 3-channel format. It has the same patient folder structure as the original dicom set.\n\nLet's take a look.","metadata":{}},{"cell_type":"code","source":"png_files = get_sorted_patient_files(PNG_DIR)\nprint(f'Found {len(png_files)} patients')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_range = list(range(30,34))\nslice_range = list(range(10,50,10))\n\n_, axs = plt.subplots(\n    len(slice_range),\n    len(patient_range),\n    figsize=(len(slice_range)*4,len(patient_range)*4))\n\nfor col, patient_index in enumerate(patient_range):\n    patient_id = os.path.basename(os.path.dirname(png_files[patient_index][0]))\n    axs[0,col].set_title(patient_id)\n    \n    for row, slice_index in enumerate(slice_range):\n        if slice_index < len(png_files[patient_index]):\n            axs[row,col].imshow(plt.imread(png_files[patient_index][slice_index]))\n        if col==0: axs[row,col].set_ylabel(f'Slice index {slice_index}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}