{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Goal of the competition:\n\n\nThe aim of this competition is to predict a patient’s severity of decline in lung function based on a CT scan of their lungs. Lung function is assessed based on output from a spirometer, which measures the forced vital capacity (FVC), i.e. the volume of air exhaled.\n\n\n\n# File given:\n\n- train.csv - the training set, contains full history of clinical information\n- test.csv - the test set, contains only the baseline measurement\n- train/ - contains the training patients' baseline CT scan in DICOM format\n- test/ - contains the test patients' baseline CT scan in DICOM format\n- sample_submission.csv - demonstrates the submission format\n\n\n# Columns: \n\n- Patient- a unique Id for each patient (also the name of the patient's DICOM folder)\n- Weeks- the relative number of weeks pre/post the baseline CT (may be negative)\n- FVC - the recorded lung capacity in ml\n- Percent- a computed field which approximates the patient's FVC as a percent of the typical FVC for a person of similar characteristics\n- Age\n- Sex\n- SmokingStatus"},{"metadata":{"trusted":true},"cell_type":"code","source":"# let us install gdcm library \n!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# For example, here's several helpful packages to load\nimport pandas as pd \nimport numpy as np \n#plotly\n!pip install chart_studio\nimport plotly.express as px\nimport chart_studio.plotly as py\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\nimport matplotlib.pyplot as plt\n#color\nfrom colorama import Fore, Back, Style\n\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\n\n#pydicom\nimport pydicom\n\n# Suppress warnings \nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Settings for pretty nice plots\nplt.style.use('fivethirtyeight')\nplt.show()\n\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport pydicom as dcm\nimport plotly.express as px\n\nimport seaborn as sns\nimport glob\nimport gdcm\nfrom matplotlib import animation, rc\n%matplotlib inline\nimport matplotlib\n%matplotlib inline\nmatplotlib.use(\"Agg\")\n\nimport matplotlib.animation as animation\n\nTRAIN_DIR = \"../input/osic-pulmonary-fibrosis-progression/train/\"\nfiles = glob.glob('../input/osic-pulmonary-fibrosis-progression/train/*/*/*.dcm')\n\nrc('animation', html='jshtml')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly\nplotly.offline.init_notebook_mode (connected = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')\ntest = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bar_plot(column_name):\n    ds = train[column_name].value_counts().reset_index()\n    ds.columns = ['Values', 'Total Number']\n    fig = px.bar(\n        ds, \n        x='Values', \n        y=\"Total Number\", \n        orientation='v',\n        title='Bar plot of: ' + column_name,\n        width=700,\n        height=400\n    )\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col = train.columns\ncol","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bar_plot('Weeks')\nbar_plot('Sex')\nbar_plot('SmokingStatus')\nbar_plot('Age')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train, x=\"FVC\", y=\"Age\", color='Sex')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train, x=\"FVC\", y=\"Percent\", color='Age')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train, x=\"FVC\", y=\"Weeks\", color='SmokingStatus')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(train, x=\"Weeks\", y=\"Age\", color='Sex')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfig = px.violin(train, y='Percent', x='SmokingStatus', box=True, color='Sex', points=\"all\",\n          hover_data=train.columns)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.box(train, x=\"Sex\", y=\"Age\", points=\"all\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let us check correlation:"},{"metadata":{"trusted":true},"cell_type":"code","source":"corr = train.corr()\ncorr.style.background_gradient(cmap='coolwarm')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let us see the scans and save it as gif\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"scans = glob.glob('/kaggle/input/osic-pulmonary-fibrosis-progression/train/*/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_scan(path):\n    fragments = glob.glob(path + '/*')\n    \n    slices = []\n    for f in fragments:\n        img = dcm.dcmread(f)\n        img_data = img.pixel_array\n        length = int(img.InstanceNumber)\n        slices.append((length, img_data))\n    slices.sort()\n    return [s[1] for s in slices]\n\ndef animate(ims):\n    fig = plt.figure(figsize=(11,11))\n    plt.axis('off')\n    im = plt.imshow(ims[0], cmap='gray')\n\n    def animate_func(i):\n        im.set_array(ims[i])\n        return [im]\n\n    anim = animation.FuncAnimation(fig, animate_func, frames = len(ims), interval = 1000//24)\n    \n    return anim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"movie = animate(read_scan(scans[12]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"movie","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize the scans"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\n# https://www.kaggle.com/yeayates21/osic-simple-image-eda\n\nimdir = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00123637202217151272140\"\nprint(\"total images for patient ID00123637202217151272140: \", len(os.listdir(imdir)))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# view first (columns*rows) images in order\nfig=plt.figure(figsize=(12, 12))\ncolumns = 4\nrows = 5\nimglist = os.listdir(imdir)\nfor i in range(1, columns*rows +1):\n    filename = imdir + \"/\" + str(i) + \".dcm\"\n    ds = pydicom.dcmread(filename)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(ds.pixel_array)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}