{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        continue\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport json\nfrom pathlib import Path\nfrom glob import glob\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.python.keras.preprocessing.image import load_img, img_to_array\n\nfrom keras import models, regularizers, layers, optimizers, losses, metrics\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.utils import np_utils, to_categorical\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.preprocessing import image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression/')\ntrain_path = data_path / 'train'\ntest_path = data_path / 'test'\nprint(\"training_path\", train_path)\nprint(\"test_path\", test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input/osic-pulmonary-fibrosis-progression/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read DICOM"},{"metadata":{},"cell_type":"markdown","source":"## What is DICOM?\n\nDICOM (Digital Imaging and Communications in Medicine) is a format that has metadata, as well as Pixeldata attached to it. \nSome basic info can be extracted from image metadata: gender and age of the patient, as well as info how the image is sampled and generated. "},{"metadata":{},"cell_type":"markdown","source":"## Import pydicom\nPydicom is a Python package specifically for parsing .dcm files. "},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nfrom pydicom.data import get_testdata_files\n\nprint(__doc__)\n\nPathDicom = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\nlstFilesDCM = []  # create an empty list\nfor dirName, subdirList, fileList in os.walk(PathDicom):\n    for filename in fileList:\n        if \".dcm\" in filename.lower():  # check whether the file's DICOM\n            lstFilesDCM.append(os.path.join(dirName,filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(lstFilesDCM[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"RefDs = pydicom.dcmread(lstFilesDCM[0])\nRefDs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get ref file\nRefDs = pydicom.dcmread(lstFilesDCM[0])\n\n# Load dimensions based on the number of rows, columns, and slices (along the Z axis)\nConstPixelDims = (int(RefDs.Rows), int(RefDs.Columns), len(lstFilesDCM))\nprint(ConstPixelDims)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Look metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"pat_name = RefDs.PatientName\ndisplay_name = pat_name.family_name + \", \" + pat_name.given_name\nprint(\"Patient's name................:\", display_name)\nprint(\"Patient id....................:\", RefDs.PatientID)\nprint(\"Modality......................:\", RefDs.Modality)\nprint(\"BodyPartExamined..............:\", RefDs.BodyPartExamined)  \nprint(\"Image Position    (Patient)...:\", RefDs.ImagePositionPatient)\nprint(\"Image Orientation (Patient)...:\", RefDs.ImageOrientationPatient)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot an image"},{"metadata":{"trusted":true},"cell_type":"code","source":"if 'PixelData' in RefDs:\n    rows = int(RefDs.Rows)\n    cols = int(RefDs.Columns)\n    print(\"Image size.......: {rows:d} x {cols:d}, {size:d} bytes\".format(rows=rows, cols=cols, size=len(RefDs.PixelData)))\n    if 'PixelSpacing' in RefDs:\n        print(\"Pixel spacing....:\", RefDs.PixelSpacing) \n\n        \n# use .get() if not sure the item exists, and want a default value if missing\nprint(\"Slice location...:\", RefDs.get('SliceLocation', \"(missing)\"))\n\n# plot the image using matplotlib\nplt.imshow(RefDs.pixel_array, cmap=plt.cm.bone)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make it bigger easily"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10,10))\nplt.imshow(RefDs.pixel_array, cmap=plt.cm.bone)\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot few images"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(data_path / 'test.csv')\ndata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.loc[data['Patient'] == 'ID00422637202311677017371']\n#Age is 73, Male","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#PATH = '/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00422637202311677017371/'\n\ndim = 25\nsize = 6\ndata2Dlist = [[0 for x in range(size)] for y in range(dim)] \n\nfor i in range(0,dim):\n    print(i, lstFilesDCM[i])\n    data2Dlist[i][0] = pydicom.dcmread(lstFilesDCM[i]).PatientID\n    data2Dlist[i][1] = pydicom.dcmread(lstFilesDCM[i]).Modality\n    data2Dlist[i][2] = pydicom.dcmread(lstFilesDCM[i]).BodyPartExamined\n    data2Dlist[i][3] = pydicom.dcmread(lstFilesDCM[i]).InstanceNumber\n    data2Dlist[i][4] = '73'\n    data2Dlist[i][5] = 'male'\n                      \n\ndata2Dlist\ndf = pd.DataFrame(data2Dlist, columns=['ID', 'Modality', 'BPE', 'Slice', 'Age', 'Sex'])\ndf\n                      \n##\n#data = pd.DataFrame([{'ID': pydicom.dcmread(lstFilesDCM[i]).PatientID}])\n#data['Modality'] = pydicom.dcmread(lstFilesDCM[i]).Modality\n#data['BPE'] = pydicom.dcmread(lstFilesDCM[i]).BodyPartExamined\n#data['Num'] = pydicom.dcmread(lstFilesDCM[i]).InstanceNumber","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot in different way"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap=plt.cm.bone) \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images.png',dpi=300)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Explore classes of colormaps\n* Sequential\n* Sequential2\n* Diverging\n* Cyclic\n* Qualitative\n* Miscellaneous"},{"metadata":{"trusted":true},"cell_type":"code","source":"## The lightness parameters from 'Sequential' class increases monotonically through the colormaps. \n## The 'inferno' is from 'Perceptually Uniform Sequential' subclass and has nice visualization. \n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='inferno') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_inferno.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## The lightness parameters from 'Sequential' class increases monotonically through the colormaps. \n## In Sequential colormaps, different colors can be tried: \n## 'Greys', 'Purples', 'Blues', 'Greens', 'Oranges', 'Reds', 'YlOrBr', 'YlOrRd', 'OrRd', 'PuRd', 'RdPu', 'BuPu', 'GnBu', 'PuBu', 'YlGnBu', 'PuBuGn', 'BuGn', 'YlGn'. \n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='Reds') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_reds.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Some lightness parameters from 'Sequential2' class are monotonically increasing (ex: binary)\n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='binary') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_binary.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Some lightness parameters from 'Sequential2' class have kinks (ex: hot)\n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='hot') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_hot.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## The lightness parameters from 'Diverging' class are monotonically increasing up to a maximum and follows by monotonically decreasing values. \n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='Spectral') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_spectral.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## The lightness parameters from 'Diverging' class are monotonically increasing up to a maximum and follows by monotonically decreasing values. \n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='seismic') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_seismic.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## The lightness parameters from 'Cyclic' class change monotonically from start to middle, and inversely from middle to end.\n## In Cyclic colormaps we want to start and end on the same color, and meet a symmetric center point in the middle. \n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='twilight') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_twilight.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## The lightness parameters values in 'Qualitative' move all over the place throughout the colormap, and are clearly not monotonically increasing. \n## These would not be good options for use as perceptual colormaps.\n## Qualitative colormaps are not aimed at being perceptual maps, but looking at the lightness parameter can verify that for us. \n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='Dark2') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_dark2.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Some of the miscellaneous colormaps have particular uses for which they have been created.\n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='gnuplot2') #gnuplot, gnuplot2\n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_gnuplot2.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Some of the miscellaneous colormaps have particular uses for which they have been created. \n## CMRmap was created to convert well to grayscale, though it does appear to have some small kinks in lightness parameters.\n\nfig, axs = plt.subplots(5,5, figsize=(23,23))\n\nfor i in range(0,25):\n    RefDs = pydicom.dcmread(lstFilesDCM[i])\n    axs[i//5, i%5].imshow(RefDs.pixel_array, cmap='CMRmap') \n    axs[i//5, i%5].set_title('Modality: {} BPE: {}\\n Slice: {} Age: {} Sex: {}'.format(df.Modality[i],df.BPE[i],df.Slice[i],df.Age[i],df.Sex[i]))\n    \nplt.savefig('data_dicom_few_images_CMRmap.png',dpi=100)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Explore tables"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(data_path / 'train.csv')\ntest  = pd.read_csv(data_path / 'test.csv')\nsub   = pd.read_csv(data_path / 'sample_submission.csv')\n\ntrain.shape, test.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train:\\n',train.head(5),'\\n')\nprint(train.isna().sum())\nprint('\\n---------------------------------------------------------------------------\\n')\nprint('Test:\\n',train.head(5),'\\n')\nprint(test.isna().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(6,2, figsize=(14,28))\n\ntrain['Sex'].value_counts().plot(kind='bar', legend=True, ax=axs[0,0])\ntest['Sex'].value_counts().plot(kind='bar', legend=True, ax=axs[0,1])\n\ntrain['Age'].hist(bins=50, ax=axs[1,0])\ntest['Age'].hist(bins=50, ax=axs[1,1])\naxs[1,0].set_xlabel('Age')\naxs[1,1].set_xlabel('Age')\n\ntrain['SmokingStatus'].value_counts().plot(kind='bar', legend=True, ax=axs[2,0])\ntest['SmokingStatus'].value_counts().plot(kind='bar', legend=True, ax=axs[2,1])\n\ntrain['Percent'].hist(bins=50, ax=axs[3,0])\ntest['Percent'].hist(bins=50, ax=axs[3,1])\naxs[3,0].set_xlabel('Percent')\naxs[3,1].set_xlabel('Percent')\n\ntrain['FVC'].hist(bins=20, ax=axs[4,0])\ntest['FVC'].hist(bins=20, ax=axs[4,1])\naxs[4,0].set_xlabel('FVC')\naxs[4,1].set_xlabel('FVC')\n\ntrain['Weeks'].hist(bins=20, ax=axs[5,0])\ntest['Weeks'].hist(bins=20, ax=axs[5,1])\naxs[5,0].set_xlabel('Weeks')\naxs[5,1].set_xlabel('Weeks')\n\nplt.savefig('data_sex_age_smokingstatus_percent_fvc_weeks.png',dpi=300)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(test)):\n    sub.loc[sub['Patient_Week'].str.contains(test.Patient[i]), 'FVC'] = test.FVC[i]\n    \nsub.to_csv('submission.csv', index=False)\nprint(\"file in ready!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}