{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-22T18:51:53.255747Z","iopub.execute_input":"2022-12-22T18:51:53.256348Z","iopub.status.idle":"2022-12-22T18:52:02.897766Z","shell.execute_reply.started":"2022-12-22T18:51:53.256215Z","shell.execute_reply":"2022-12-22T18:52:02.895870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directory = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\ntrain_df = pd.read_csv(directory + '/train.csv')\ntest_df = pd.read_csv(directory + '/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:02.901101Z","iopub.execute_input":"2022-12-22T18:52:02.901738Z","iopub.status.idle":"2022-12-22T18:52:02.942475Z","shell.execute_reply.started":"2022-12-22T18:52:02.901678Z","shell.execute_reply":"2022-12-22T18:52:02.940860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:02.944764Z","iopub.execute_input":"2022-12-22T18:52:02.945337Z","iopub.status.idle":"2022-12-22T18:52:02.989263Z","shell.execute_reply.started":"2022-12-22T18:52:02.945288Z","shell.execute_reply":"2022-12-22T18:52:02.987964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:36:50.844497Z","iopub.execute_input":"2022-12-22T19:36:50.844962Z","iopub.status.idle":"2022-12-22T19:36:50.861866Z","shell.execute_reply.started":"2022-12-22T19:36:50.844925Z","shell.execute_reply":"2022-12-22T19:36:50.860503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Patient'].count()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:33:36.307762Z","iopub.execute_input":"2022-12-22T19:33:36.308563Z","iopub.status.idle":"2022-12-22T19:33:36.321454Z","shell.execute_reply.started":"2022-12-22T19:33:36.308503Z","shell.execute_reply":"2022-12-22T19:33:36.320290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Patient'].count()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:34:16.857396Z","iopub.execute_input":"2022-12-22T19:34:16.858864Z","iopub.status.idle":"2022-12-22T19:34:16.868700Z","shell.execute_reply.started":"2022-12-22T19:34:16.858803Z","shell.execute_reply":"2022-12-22T19:34:16.867383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n\nlen(list(Path(directory+'/train/').rglob(\"*\")))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:34:54.472473Z","iopub.execute_input":"2022-12-22T19:34:54.473782Z","iopub.status.idle":"2022-12-22T19:34:55.105010Z","shell.execute_reply.started":"2022-12-22T19:34:54.473732Z","shell.execute_reply":"2022-12-22T19:34:55.103117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(Path(directory+'/test/').rglob(\"*\")))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:35:16.596015Z","iopub.execute_input":"2022-12-22T19:35:16.596539Z","iopub.status.idle":"2022-12-22T19:35:16.630127Z","shell.execute_reply.started":"2022-12-22T19:35:16.596499Z","shell.execute_reply":"2022-12-22T19:35:16.628630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:02.992055Z","iopub.execute_input":"2022-12-22T18:52:02.992642Z","iopub.status.idle":"2022-12-22T18:52:03.006749Z","shell.execute_reply.started":"2022-12-22T18:52:02.992589Z","shell.execute_reply":"2022-12-22T18:52:03.004999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:37:58.915281Z","iopub.execute_input":"2022-12-22T19:37:58.915829Z","iopub.status.idle":"2022-12-22T19:37:58.928772Z","shell.execute_reply.started":"2022-12-22T19:37:58.915791Z","shell.execute_reply":"2022-12-22T19:37:58.927795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:03.008351Z","iopub.execute_input":"2022-12-22T18:52:03.008801Z","iopub.status.idle":"2022-12-22T18:52:03.026527Z","shell.execute_reply.started":"2022-12-22T18:52:03.008760Z","shell.execute_reply":"2022-12-22T18:52:03.024943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:38:28.734325Z","iopub.execute_input":"2022-12-22T19:38:28.734866Z","iopub.status.idle":"2022-12-22T19:38:28.742880Z","shell.execute_reply.started":"2022-12-22T19:38:28.734822Z","shell.execute_reply":"2022-12-22T19:38:28.742103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:03.028499Z","iopub.execute_input":"2022-12-22T18:52:03.029002Z","iopub.status.idle":"2022-12-22T18:52:03.035742Z","shell.execute_reply.started":"2022-12-22T18:52:03.028952Z","shell.execute_reply":"2022-12-22T18:52:03.034439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:39:12.005615Z","iopub.execute_input":"2022-12-22T19:39:12.006043Z","iopub.status.idle":"2022-12-22T19:39:12.018561Z","shell.execute_reply.started":"2022-12-22T19:39:12.006011Z","shell.execute_reply":"2022-12-22T19:39:12.017250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].hist(bins=3)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:03.434874Z","iopub.execute_input":"2022-12-22T18:52:03.435543Z","iopub.status.idle":"2022-12-22T18:52:03.581938Z","shell.execute_reply.started":"2022-12-22T18:52:03.435502Z","shell.execute_reply":"2022-12-22T18:52:03.580330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].hist(bins=3)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:43:03.342327Z","iopub.execute_input":"2022-12-22T19:43:03.342799Z","iopub.status.idle":"2022-12-22T19:43:03.560038Z","shell.execute_reply.started":"2022-12-22T19:43:03.342766Z","shell.execute_reply":"2022-12-22T19:43:03.558832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['Age'].hist(bins=20,histtype='step')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:44:51.376056Z","iopub.execute_input":"2022-12-22T19:44:51.376597Z","iopub.status.idle":"2022-12-22T19:44:51.641675Z","shell.execute_reply.started":"2022-12-22T19:44:51.376557Z","shell.execute_reply":"2022-12-22T19:44:51.640200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:46:32.100952Z","iopub.execute_input":"2022-12-22T19:46:32.101565Z","iopub.status.idle":"2022-12-22T19:46:32.118919Z","shell.execute_reply.started":"2022-12-22T19:46:32.101510Z","shell.execute_reply":"2022-12-22T19:46:32.117262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Percent'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:46:56.878392Z","iopub.execute_input":"2022-12-22T19:46:56.878831Z","iopub.status.idle":"2022-12-22T19:46:56.891866Z","shell.execute_reply.started":"2022-12-22T19:46:56.878796Z","shell.execute_reply":"2022-12-22T19:46:56.890707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['Age'].hist(bins=10,histtype='step')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:47:26.605443Z","iopub.execute_input":"2022-12-22T19:47:26.605968Z","iopub.status.idle":"2022-12-22T19:47:26.852031Z","shell.execute_reply.started":"2022-12-22T19:47:26.605928Z","shell.execute_reply":"2022-12-22T19:47:26.850494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].hist(bins=30)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:03.583866Z","iopub.execute_input":"2022-12-22T18:52:03.584557Z","iopub.status.idle":"2022-12-22T18:52:03.904479Z","shell.execute_reply.started":"2022-12-22T18:52:03.584506Z","shell.execute_reply":"2022-12-22T18:52:03.903105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].hist(bins=20)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:03.908555Z","iopub.execute_input":"2022-12-22T18:52:03.909075Z","iopub.status.idle":"2022-12-22T18:52:04.405075Z","shell.execute_reply.started":"2022-12-22T18:52:03.909033Z","shell.execute_reply":"2022-12-22T18:52:04.403728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:04.406900Z","iopub.execute_input":"2022-12-22T18:52:04.407456Z","iopub.status.idle":"2022-12-22T18:52:04.424162Z","shell.execute_reply.started":"2022-12-22T18:52:04.407403Z","shell.execute_reply":"2022-12-22T18:52:04.422288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['Age'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:04.426648Z","iopub.execute_input":"2022-12-22T18:52:04.427154Z","iopub.status.idle":"2022-12-22T18:52:04.677857Z","shell.execute_reply.started":"2022-12-22T18:52:04.427103Z","shell.execute_reply":"2022-12-22T18:52:04.676416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:04.680341Z","iopub.execute_input":"2022-12-22T18:52:04.680929Z","iopub.status.idle":"2022-12-22T18:52:04.687744Z","shell.execute_reply.started":"2022-12-22T18:52:04.680874Z","shell.execute_reply":"2022-12-22T18:52:04.686241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(Path(directory+'/train/').rglob(\"*\")))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:04.689888Z","iopub.execute_input":"2022-12-22T18:52:04.690478Z","iopub.status.idle":"2022-12-22T18:52:05.202377Z","shell.execute_reply.started":"2022-12-22T18:52:04.690418Z","shell.execute_reply":"2022-12-22T18:52:05.201342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(Path(directory+'/test/').rglob(\"*\")))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:05.203763Z","iopub.execute_input":"2022-12-22T18:52:05.204124Z","iopub.status.idle":"2022-12-22T18:52:05.228502Z","shell.execute_reply.started":"2022-12-22T18:52:05.204091Z","shell.execute_reply":"2022-12-22T18:52:05.227239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['Age'].hist(bins=10,histtype='step')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:05.230515Z","iopub.execute_input":"2022-12-22T18:52:05.231397Z","iopub.status.idle":"2022-12-22T18:52:05.472893Z","shell.execute_reply.started":"2022-12-22T18:52:05.231352Z","shell.execute_reply":"2022-12-22T18:52:05.471249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['Age'].hist(bins=10,histtype='step')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:05.474745Z","iopub.execute_input":"2022-12-22T18:52:05.475913Z","iopub.status.idle":"2022-12-22T18:52:05.907921Z","shell.execute_reply.started":"2022-12-22T18:52:05.475865Z","shell.execute_reply":"2022-12-22T18:52:05.906268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\n\ndef extract_dicom_meta_data(filename: str) -> Dict:\n    # Load image\n    \n    image_data = pydicom.read_file(filename)\n    img=np.array(image_data.pixel_array).flatten()\n    row = {\n        'Patient': image_data.PatientID,\n        'body_part_examined': image_data.BodyPartExamined,\n        'image_position_patient': image_data.ImagePositionPatient,\n        'image_orientation_patient': image_data.ImageOrientationPatient,\n        'photometric_interpretation': image_data.PhotometricInterpretation,\n        'rows': image_data.Rows,\n        'columns': image_data.Columns,\n        'pixel_spacing': image_data.PixelSpacing,\n        'window_center': image_data.WindowCenter,\n        'window_width': image_data.WindowWidth,\n        'modality': image_data.Modality,\n        'StudyInstanceUID': image_data.StudyInstanceUID,\n        'SeriesInstanceUID': image_data.StudyInstanceUID,\n        'StudyID': image_data.StudyInstanceUID, \n        'SamplesPerPixel': image_data.SamplesPerPixel,\n        'BitsAllocated': image_data.BitsAllocated,\n        'BitsStored': image_data.BitsStored,\n        'HighBit': image_data.HighBit,\n        'PixelRepresentation': image_data.PixelRepresentation,\n        'RescaleIntercept': image_data.RescaleIntercept,\n        'RescaleSlope': image_data.RescaleSlope,\n        'img_min': np.min(img),\n        'img_max': np.max(img),\n        'img_mean': np.mean(img),\n        'img_std': np.std(img)}\n\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:05.909504Z","iopub.execute_input":"2022-12-22T18:52:05.909897Z","iopub.status.idle":"2022-12-22T18:52:05.922067Z","shell.execute_reply.started":"2022-12-22T18:52:05.909863Z","shell.execute_reply":"2022-12-22T18:52:05.920841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport tqdm \nimport pydicom\n\ntrain_image_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/train'\ntrain_image_files = glob.glob(os.path.join(train_image_path, '*', '*.dcm'))\n\nmeta_data_df = []\nfor filename in tqdm.tqdm(train_image_files):\n    try:\n        meta_data_df.append(extract_dicom_meta_data(filename))\n    except Exception as e:\n        print(e)\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-12-22T18:52:05.924002Z","iopub.execute_input":"2022-12-22T18:52:05.924785Z","iopub.status.idle":"2022-12-22T19:03:12.224005Z","shell.execute_reply.started":"2022-12-22T18:52:05.924740Z","shell.execute_reply":"2022-12-22T19:03:12.222502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df = pd.DataFrame.from_dict(meta_data_df)\nmeta_data_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T19:03:12.226300Z","iopub.execute_input":"2022-12-22T19:03:12.226748Z","iopub.status.idle":"2022-12-22T19:03:12.818869Z","shell.execute_reply.started":"2022-12-22T19:03:12.226709Z","shell.execute_reply":"2022-12-22T19:03:12.817470Z"},"trusted":true},"execution_count":null,"outputs":[]}]}