{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-22T16:13:21.775454Z","iopub.execute_input":"2022-12-22T16:13:21.775936Z","iopub.status.idle":"2022-12-22T16:13:26.972500Z","shell.execute_reply.started":"2022-12-22T16:13:21.775897Z","shell.execute_reply":"2022-12-22T16:13:26.970321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nimport random\nimport math\nimport matplotlib\nfrom termcolor import colored\nimport os\nfrom os import listdir\nfrom os.path import join, getsize\nimport glob\nimport cv2\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.offline import iplot\nimport cufflinks\ncufflinks.go_offline()\ncufflinks.set_config_file(world_readable=True, theme='pearl')\n\nfrom skimage import measure\nfrom skimage.morphology import disk, opening, closing\n\nimport tensorflow as tf\nfrom tensorflow.keras import Model\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import train_test_split, KFold\n\nfrom tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)\n\n%matplotlib inline\n\nsns.set(style='darkgrid', palette='Set2')\n \nimport warnings\nwarnings.filterwarnings('ignore')\n\nplt.style.use('fivethirtyeight')\nplt.show()\n\nimport pydicom","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:26.974808Z","iopub.execute_input":"2022-12-22T16:13:26.976105Z","iopub.status.idle":"2022-12-22T16:13:27.011165Z","shell.execute_reply.started":"2022-12-22T16:13:26.976042Z","shell.execute_reply":"2022-12-22T16:13:27.009912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport pydicom\nimport glob\nimport imageio\nfrom IPython.display import Image","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.013094Z","iopub.execute_input":"2022-12-22T16:13:27.013725Z","iopub.status.idle":"2022-12-22T16:13:27.025848Z","shell.execute_reply.started":"2022-12-22T16:13:27.013653Z","shell.execute_reply":"2022-12-22T16:13:27.024531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\n\nimport pydicom\nfrom skimage.transform import resize\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.028818Z","iopub.execute_input":"2022-12-22T16:13:27.029234Z","iopub.status.idle":"2022-12-22T16:13:27.039759Z","shell.execute_reply.started":"2022-12-22T16:13:27.029202Z","shell.execute_reply":"2022-12-22T16:13:27.038443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_df = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.041326Z","iopub.execute_input":"2022-12-22T16:13:27.041987Z","iopub.status.idle":"2022-12-22T16:13:27.077381Z","shell.execute_reply.started":"2022-12-22T16:13:27.041949Z","shell.execute_reply":"2022-12-22T16:13:27.076048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.079647Z","iopub.execute_input":"2022-12-22T16:13:27.080444Z","iopub.status.idle":"2022-12-22T16:13:27.095117Z","shell.execute_reply.started":"2022-12-22T16:13:27.080391Z","shell.execute_reply":"2022-12-22T16:13:27.093526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.100092Z","iopub.execute_input":"2022-12-22T16:13:27.100481Z","iopub.status.idle":"2022-12-22T16:13:27.117054Z","shell.execute_reply.started":"2022-12-22T16:13:27.100449Z","shell.execute_reply":"2022-12-22T16:13:27.115899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.118565Z","iopub.execute_input":"2022-12-22T16:13:27.119042Z","iopub.status.idle":"2022-12-22T16:13:27.133882Z","shell.execute_reply.started":"2022-12-22T16:13:27.119009Z","shell.execute_reply":"2022-12-22T16:13:27.132600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.135568Z","iopub.execute_input":"2022-12-22T16:13:27.136056Z","iopub.status.idle":"2022-12-22T16:13:27.150910Z","shell.execute_reply.started":"2022-12-22T16:13:27.136009Z","shell.execute_reply":"2022-12-22T16:13:27.149353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.157743Z","iopub.execute_input":"2022-12-22T16:13:27.158176Z","iopub.status.idle":"2022-12-22T16:13:27.169838Z","shell.execute_reply.started":"2022-12-22T16:13:27.158140Z","shell.execute_reply":"2022-12-22T16:13:27.168433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Weeks'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.173761Z","iopub.execute_input":"2022-12-22T16:13:27.174131Z","iopub.status.idle":"2022-12-22T16:13:27.599080Z","shell.execute_reply.started":"2022-12-22T16:13:27.174100Z","shell.execute_reply":"2022-12-22T16:13:27.597546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Weeks'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.600877Z","iopub.execute_input":"2022-12-22T16:13:27.601257Z","iopub.status.idle":"2022-12-22T16:13:27.614575Z","shell.execute_reply.started":"2022-12-22T16:13:27.601222Z","shell.execute_reply":"2022-12-22T16:13:27.613092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['FVC'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:27.616473Z","iopub.execute_input":"2022-12-22T16:13:27.616902Z","iopub.status.idle":"2022-12-22T16:13:28.011235Z","shell.execute_reply.started":"2022-12-22T16:13:27.616864Z","shell.execute_reply":"2022-12-22T16:13:28.009844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['FVC'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:28.013416Z","iopub.execute_input":"2022-12-22T16:13:28.013843Z","iopub.status.idle":"2022-12-22T16:13:28.025476Z","shell.execute_reply.started":"2022-12-22T16:13:28.013804Z","shell.execute_reply":"2022-12-22T16:13:28.024135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Percent'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:28.027467Z","iopub.execute_input":"2022-12-22T16:13:28.027916Z","iopub.status.idle":"2022-12-22T16:13:28.426282Z","shell.execute_reply.started":"2022-12-22T16:13:28.027880Z","shell.execute_reply":"2022-12-22T16:13:28.424777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Percent'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:28.428533Z","iopub.execute_input":"2022-12-22T16:13:28.429060Z","iopub.status.idle":"2022-12-22T16:13:28.444395Z","shell.execute_reply.started":"2022-12-22T16:13:28.429010Z","shell.execute_reply":"2022-12-22T16:13:28.443261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Age'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:28.446128Z","iopub.execute_input":"2022-12-22T16:13:28.446524Z","iopub.status.idle":"2022-12-22T16:13:28.841535Z","shell.execute_reply.started":"2022-12-22T16:13:28.446479Z","shell.execute_reply":"2022-12-22T16:13:28.840234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Age'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:28.843075Z","iopub.execute_input":"2022-12-22T16:13:28.843450Z","iopub.status.idle":"2022-12-22T16:13:28.854718Z","shell.execute_reply.started":"2022-12-22T16:13:28.843419Z","shell.execute_reply":"2022-12-22T16:13:28.853765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Sex'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:28.856011Z","iopub.execute_input":"2022-12-22T16:13:28.856712Z","iopub.status.idle":"2022-12-22T16:13:29.158645Z","shell.execute_reply.started":"2022-12-22T16:13:28.856653Z","shell.execute_reply":"2022-12-22T16:13:29.157155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Sex'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.160075Z","iopub.execute_input":"2022-12-22T16:13:29.160431Z","iopub.status.idle":"2022-12-22T16:13:29.175313Z","shell.execute_reply.started":"2022-12-22T16:13:29.160399Z","shell.execute_reply":"2022-12-22T16:13:29.173928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['SmokingStatus'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.177012Z","iopub.execute_input":"2022-12-22T16:13:29.177518Z","iopub.status.idle":"2022-12-22T16:13:29.498382Z","shell.execute_reply.started":"2022-12-22T16:13:29.177468Z","shell.execute_reply":"2022-12-22T16:13:29.497000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['SmokingStatus'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.499578Z","iopub.execute_input":"2022-12-22T16:13:29.499943Z","iopub.status.idle":"2022-12-22T16:13:29.512460Z","shell.execute_reply.started":"2022-12-22T16:13:29.499909Z","shell.execute_reply":"2022-12-22T16:13:29.511069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Weeks')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.514357Z","iopub.execute_input":"2022-12-22T16:13:29.514899Z","iopub.status.idle":"2022-12-22T16:13:29.531428Z","shell.execute_reply.started":"2022-12-22T16:13:29.514849Z","shell.execute_reply":"2022-12-22T16:13:29.530159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Weeks')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.532504Z","iopub.execute_input":"2022-12-22T16:13:29.532886Z","iopub.status.idle":"2022-12-22T16:13:29.828522Z","shell.execute_reply.started":"2022-12-22T16:13:29.532854Z","shell.execute_reply":"2022-12-22T16:13:29.827231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Percent')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.829816Z","iopub.execute_input":"2022-12-22T16:13:29.830859Z","iopub.status.idle":"2022-12-22T16:13:29.847759Z","shell.execute_reply.started":"2022-12-22T16:13:29.830817Z","shell.execute_reply":"2022-12-22T16:13:29.846275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Percent')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:29.849389Z","iopub.execute_input":"2022-12-22T16:13:29.849835Z","iopub.status.idle":"2022-12-22T16:13:30.160454Z","shell.execute_reply.started":"2022-12-22T16:13:29.849797Z","shell.execute_reply":"2022-12-22T16:13:30.159369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Age')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:30.162086Z","iopub.execute_input":"2022-12-22T16:13:30.162804Z","iopub.status.idle":"2022-12-22T16:13:30.178281Z","shell.execute_reply.started":"2022-12-22T16:13:30.162766Z","shell.execute_reply":"2022-12-22T16:13:30.177167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Age')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:30.187556Z","iopub.execute_input":"2022-12-22T16:13:30.188359Z","iopub.status.idle":"2022-12-22T16:13:30.481654Z","shell.execute_reply.started":"2022-12-22T16:13:30.188321Z","shell.execute_reply":"2022-12-22T16:13:30.480303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Sex')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:30.483034Z","iopub.execute_input":"2022-12-22T16:13:30.483383Z","iopub.status.idle":"2022-12-22T16:13:30.499866Z","shell.execute_reply.started":"2022-12-22T16:13:30.483352Z","shell.execute_reply":"2022-12-22T16:13:30.498296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Sex')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:30.502070Z","iopub.execute_input":"2022-12-22T16:13:30.502447Z","iopub.status.idle":"2022-12-22T16:13:30.701626Z","shell.execute_reply.started":"2022-12-22T16:13:30.502414Z","shell.execute_reply":"2022-12-22T16:13:30.700504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('SmokingStatus')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:30.702829Z","iopub.execute_input":"2022-12-22T16:13:30.703794Z","iopub.status.idle":"2022-12-22T16:13:30.718744Z","shell.execute_reply.started":"2022-12-22T16:13:30.703758Z","shell.execute_reply":"2022-12-22T16:13:30.717402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('SmokingStatus')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:30.720284Z","iopub.execute_input":"2022-12-22T16:13:30.720783Z","iopub.status.idle":"2022-12-22T16:13:34.892038Z","shell.execute_reply.started":"2022-12-22T16:13:30.720745Z","shell.execute_reply":"2022-12-22T16:13:34.890658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:34.893598Z","iopub.execute_input":"2022-12-22T16:13:34.893970Z","iopub.status.idle":"2022-12-22T16:13:34.916699Z","shell.execute_reply.started":"2022-12-22T16:13:34.893934Z","shell.execute_reply":"2022-12-22T16:13:34.915491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:34.918323Z","iopub.execute_input":"2022-12-22T16:13:34.918753Z","iopub.status.idle":"2022-12-22T16:13:34.934821Z","shell.execute_reply.started":"2022-12-22T16:13:34.918713Z","shell.execute_reply":"2022-12-22T16:13:34.933605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:34.936641Z","iopub.execute_input":"2022-12-22T16:13:34.937035Z","iopub.status.idle":"2022-12-22T16:13:34.944984Z","shell.execute_reply.started":"2022-12-22T16:13:34.936998Z","shell.execute_reply":"2022-12-22T16:13:34.943680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:34.946482Z","iopub.execute_input":"2022-12-22T16:13:34.947005Z","iopub.status.idle":"2022-12-22T16:13:34.961624Z","shell.execute_reply.started":"2022-12-22T16:13:34.946969Z","shell.execute_reply":"2022-12-22T16:13:34.960126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:34.964374Z","iopub.execute_input":"2022-12-22T16:13:34.964947Z","iopub.status.idle":"2022-12-22T16:13:35.347108Z","shell.execute_reply.started":"2022-12-22T16:13:34.964888Z","shell.execute_reply":"2022-12-22T16:13:35.345827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:35.348610Z","iopub.execute_input":"2022-12-22T16:13:35.349071Z","iopub.status.idle":"2022-12-22T16:13:35.362363Z","shell.execute_reply.started":"2022-12-22T16:13:35.349034Z","shell.execute_reply":"2022-12-22T16:13:35.361070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:35.364423Z","iopub.execute_input":"2022-12-22T16:13:35.365001Z","iopub.status.idle":"2022-12-22T16:13:35.748005Z","shell.execute_reply.started":"2022-12-22T16:13:35.364935Z","shell.execute_reply":"2022-12-22T16:13:35.746543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:35.749602Z","iopub.execute_input":"2022-12-22T16:13:35.750030Z","iopub.status.idle":"2022-12-22T16:13:35.762407Z","shell.execute_reply.started":"2022-12-22T16:13:35.749981Z","shell.execute_reply":"2022-12-22T16:13:35.760738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Percent'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:35.764291Z","iopub.execute_input":"2022-12-22T16:13:35.764636Z","iopub.status.idle":"2022-12-22T16:13:36.138789Z","shell.execute_reply.started":"2022-12-22T16:13:35.764605Z","shell.execute_reply":"2022-12-22T16:13:36.137426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Percent'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:36.140656Z","iopub.execute_input":"2022-12-22T16:13:36.141541Z","iopub.status.idle":"2022-12-22T16:13:36.153385Z","shell.execute_reply.started":"2022-12-22T16:13:36.141501Z","shell.execute_reply":"2022-12-22T16:13:36.152077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:36.155545Z","iopub.execute_input":"2022-12-22T16:13:36.156040Z","iopub.status.idle":"2022-12-22T16:13:36.569929Z","shell.execute_reply.started":"2022-12-22T16:13:36.155995Z","shell.execute_reply":"2022-12-22T16:13:36.568893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:36.571431Z","iopub.execute_input":"2022-12-22T16:13:36.571814Z","iopub.status.idle":"2022-12-22T16:13:36.587738Z","shell.execute_reply.started":"2022-12-22T16:13:36.571778Z","shell.execute_reply":"2022-12-22T16:13:36.586628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:36.589393Z","iopub.execute_input":"2022-12-22T16:13:36.590245Z","iopub.status.idle":"2022-12-22T16:13:36.902368Z","shell.execute_reply.started":"2022-12-22T16:13:36.590205Z","shell.execute_reply":"2022-12-22T16:13:36.901056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:36.903815Z","iopub.execute_input":"2022-12-22T16:13:36.904228Z","iopub.status.idle":"2022-12-22T16:13:36.915074Z","shell.execute_reply.started":"2022-12-22T16:13:36.904185Z","shell.execute_reply":"2022-12-22T16:13:36.913835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:36.916433Z","iopub.execute_input":"2022-12-22T16:13:36.916842Z","iopub.status.idle":"2022-12-22T16:13:37.232569Z","shell.execute_reply.started":"2022-12-22T16:13:36.916804Z","shell.execute_reply":"2022-12-22T16:13:37.231467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.233970Z","iopub.execute_input":"2022-12-22T16:13:37.234298Z","iopub.status.idle":"2022-12-22T16:13:37.245404Z","shell.execute_reply.started":"2022-12-22T16:13:37.234268Z","shell.execute_reply":"2022-12-22T16:13:37.244035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Weeks')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.247605Z","iopub.execute_input":"2022-12-22T16:13:37.248166Z","iopub.status.idle":"2022-12-22T16:13:37.268084Z","shell.execute_reply.started":"2022-12-22T16:13:37.248126Z","shell.execute_reply":"2022-12-22T16:13:37.266899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Weeks')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.269424Z","iopub.execute_input":"2022-12-22T16:13:37.269799Z","iopub.status.idle":"2022-12-22T16:13:37.495730Z","shell.execute_reply.started":"2022-12-22T16:13:37.269764Z","shell.execute_reply":"2022-12-22T16:13:37.494786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Percent')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.497013Z","iopub.execute_input":"2022-12-22T16:13:37.498094Z","iopub.status.idle":"2022-12-22T16:13:37.518936Z","shell.execute_reply.started":"2022-12-22T16:13:37.498041Z","shell.execute_reply":"2022-12-22T16:13:37.517602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Percent')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.520440Z","iopub.execute_input":"2022-12-22T16:13:37.520813Z","iopub.status.idle":"2022-12-22T16:13:37.760090Z","shell.execute_reply.started":"2022-12-22T16:13:37.520781Z","shell.execute_reply":"2022-12-22T16:13:37.758650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Age')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.762072Z","iopub.execute_input":"2022-12-22T16:13:37.763326Z","iopub.status.idle":"2022-12-22T16:13:37.782963Z","shell.execute_reply.started":"2022-12-22T16:13:37.763273Z","shell.execute_reply":"2022-12-22T16:13:37.781701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Age')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:37.785096Z","iopub.execute_input":"2022-12-22T16:13:37.785530Z","iopub.status.idle":"2022-12-22T16:13:38.067582Z","shell.execute_reply.started":"2022-12-22T16:13:37.785493Z","shell.execute_reply":"2022-12-22T16:13:38.066193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.069226Z","iopub.execute_input":"2022-12-22T16:13:38.069567Z","iopub.status.idle":"2022-12-22T16:13:38.085662Z","shell.execute_reply.started":"2022-12-22T16:13:38.069536Z","shell.execute_reply":"2022-12-22T16:13:38.084456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.087242Z","iopub.execute_input":"2022-12-22T16:13:38.087607Z","iopub.status.idle":"2022-12-22T16:13:38.328160Z","shell.execute_reply.started":"2022-12-22T16:13:38.087574Z","shell.execute_reply":"2022-12-22T16:13:38.326923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.329478Z","iopub.execute_input":"2022-12-22T16:13:38.329897Z","iopub.status.idle":"2022-12-22T16:13:38.347391Z","shell.execute_reply.started":"2022-12-22T16:13:38.329864Z","shell.execute_reply":"2022-12-22T16:13:38.346171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.348841Z","iopub.execute_input":"2022-12-22T16:13:38.350004Z","iopub.status.idle":"2022-12-22T16:13:38.628342Z","shell.execute_reply.started":"2022-12-22T16:13:38.349963Z","shell.execute_reply":"2022-12-22T16:13:38.627367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC_pred_mean'] = train_df['FVC'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.629958Z","iopub.execute_input":"2022-12-22T16:13:38.630663Z","iopub.status.idle":"2022-12-22T16:13:38.638570Z","shell.execute_reply.started":"2022-12-22T16:13:38.630622Z","shell.execute_reply":"2022-12-22T16:13:38.637406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err'] = train_df['FVC'] - train_df['FVC_pred_mean']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.640000Z","iopub.execute_input":"2022-12-22T16:13:38.640685Z","iopub.status.idle":"2022-12-22T16:13:38.652154Z","shell.execute_reply.started":"2022-12-22T16:13:38.640632Z","shell.execute_reply":"2022-12-22T16:13:38.650530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs'] = abs(train_df['err'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.653958Z","iopub.execute_input":"2022-12-22T16:13:38.654704Z","iopub.status.idle":"2022-12-22T16:13:38.664847Z","shell.execute_reply.started":"2022-12-22T16:13:38.654635Z","shell.execute_reply":"2022-12-22T16:13:38.663226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.666463Z","iopub.execute_input":"2022-12-22T16:13:38.666939Z","iopub.status.idle":"2022-12-22T16:13:38.679310Z","shell.execute_reply.started":"2022-12-22T16:13:38.666882Z","shell.execute_reply":"2022-12-22T16:13:38.678043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs_pct'] = train_df['err_abs'] / train_df['FVC']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.680661Z","iopub.execute_input":"2022-12-22T16:13:38.681113Z","iopub.status.idle":"2022-12-22T16:13:38.689351Z","shell.execute_reply.started":"2022-12-22T16:13:38.681079Z","shell.execute_reply":"2022-12-22T16:13:38.688293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs_pct'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.690526Z","iopub.execute_input":"2022-12-22T16:13:38.690924Z","iopub.status.idle":"2022-12-22T16:13:38.702739Z","shell.execute_reply.started":"2022-12-22T16:13:38.690880Z","shell.execute_reply":"2022-12-22T16:13:38.701365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC_pred_median'] = train_df['FVC'].median()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.703895Z","iopub.execute_input":"2022-12-22T16:13:38.704280Z","iopub.status.idle":"2022-12-22T16:13:38.712248Z","shell.execute_reply.started":"2022-12-22T16:13:38.704247Z","shell.execute_reply":"2022-12-22T16:13:38.711191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.713688Z","iopub.execute_input":"2022-12-22T16:13:38.714396Z","iopub.status.idle":"2022-12-22T16:13:38.746173Z","shell.execute_reply.started":"2022-12-22T16:13:38.714360Z","shell.execute_reply":"2022-12-22T16:13:38.745035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.747937Z","iopub.execute_input":"2022-12-22T16:13:38.748293Z","iopub.status.idle":"2022-12-22T16:13:38.754349Z","shell.execute_reply.started":"2022-12-22T16:13:38.748260Z","shell.execute_reply":"2022-12-22T16:13:38.752984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_error(train_df['FVC'],train_df['FVC_pred_mean'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.756266Z","iopub.execute_input":"2022-12-22T16:13:38.757636Z","iopub.status.idle":"2022-12-22T16:13:38.766959Z","shell.execute_reply.started":"2022-12-22T16:13:38.757586Z","shell.execute_reply":"2022-12-22T16:13:38.765666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_percentage_error(train_df['FVC'],train_df['FVC_pred_mean'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.768588Z","iopub.execute_input":"2022-12-22T16:13:38.769293Z","iopub.status.idle":"2022-12-22T16:13:38.782190Z","shell.execute_reply.started":"2022-12-22T16:13:38.769245Z","shell.execute_reply":"2022-12-22T16:13:38.780771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_error(train_df['FVC'],train_df['FVC_pred_median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.783729Z","iopub.execute_input":"2022-12-22T16:13:38.784124Z","iopub.status.idle":"2022-12-22T16:13:38.793527Z","shell.execute_reply.started":"2022-12-22T16:13:38.784091Z","shell.execute_reply":"2022-12-22T16:13:38.792560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_percentage_error(train_df['FVC'],train_df['FVC_pred_median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.794630Z","iopub.execute_input":"2022-12-22T16:13:38.795021Z","iopub.status.idle":"2022-12-22T16:13:38.806537Z","shell.execute_reply.started":"2022-12-22T16:13:38.794983Z","shell.execute_reply":"2022-12-22T16:13:38.805258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.808067Z","iopub.execute_input":"2022-12-22T16:13:38.808509Z","iopub.status.idle":"2022-12-22T16:13:38.839475Z","shell.execute_reply.started":"2022-12-22T16:13:38.808461Z","shell.execute_reply":"2022-12-22T16:13:38.838175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['First_Week'] = 0\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.841183Z","iopub.execute_input":"2022-12-22T16:13:38.841690Z","iopub.status.idle":"2022-12-22T16:13:38.870563Z","shell.execute_reply.started":"2022-12-22T16:13:38.841636Z","shell.execute_reply":"2022-12-22T16:13:38.869730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition = (train_df['Weeks'] <= 1)\ntrain_df.loc[condition, 'First_Week'] = 'One Week'\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.872117Z","iopub.execute_input":"2022-12-22T16:13:38.872713Z","iopub.status.idle":"2022-12-22T16:13:38.900010Z","shell.execute_reply.started":"2022-12-22T16:13:38.872660Z","shell.execute_reply":"2022-12-22T16:13:38.898752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition = (train_df['Weeks'] > 1)\ntrain_df.loc[condition, 'First_Week'] = 'More Week'\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.901368Z","iopub.execute_input":"2022-12-22T16:13:38.901805Z","iopub.status.idle":"2022-12-22T16:13:38.934276Z","shell.execute_reply.started":"2022-12-22T16:13:38.901754Z","shell.execute_reply":"2022-12-22T16:13:38.932977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['First_Week'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.936327Z","iopub.execute_input":"2022-12-22T16:13:38.936836Z","iopub.status.idle":"2022-12-22T16:13:38.948756Z","shell.execute_reply.started":"2022-12-22T16:13:38.936786Z","shell.execute_reply":"2022-12-22T16:13:38.947234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates = train_df[train_df.duplicated(subset = ['Patient', 'Weeks'], keep = False)]\nduplicates","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.951924Z","iopub.execute_input":"2022-12-22T16:13:38.952985Z","iopub.status.idle":"2022-12-22T16:13:38.983246Z","shell.execute_reply.started":"2022-12-22T16:13:38.952928Z","shell.execute_reply":"2022-12-22T16:13:38.981441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\n\ndef extract_dicom_meta_data(filename: str) -> Dict:\n    # Load image\n    \n    image_data = pydicom.read_file(filename)\n    img=np.array(image_data.pixel_array).flatten()\n    row = {\n        'Patient': image_data.PatientID,\n        'body_part_examined': image_data.BodyPartExamined,\n        'image_position_patient': image_data.ImagePositionPatient,\n        'image_orientation_patient': image_data.ImageOrientationPatient,\n        'photometric_interpretation': image_data.PhotometricInterpretation,\n        'rows': image_data.Rows,\n        'columns': image_data.Columns,\n        'pixel_spacing': image_data.PixelSpacing,\n        'window_center': image_data.WindowCenter,\n        'window_width': image_data.WindowWidth,\n        'modality': image_data.Modality,\n        'StudyInstanceUID': image_data.StudyInstanceUID,\n        'SeriesInstanceUID': image_data.StudyInstanceUID,\n        'StudyID': image_data.StudyInstanceUID, \n        'SamplesPerPixel': image_data.SamplesPerPixel,\n        'BitsAllocated': image_data.BitsAllocated,\n        'BitsStored': image_data.BitsStored,\n        'HighBit': image_data.HighBit,\n        'PixelRepresentation': image_data.PixelRepresentation,\n        'RescaleIntercept': image_data.RescaleIntercept,\n        'RescaleSlope': image_data.RescaleSlope,\n        'img_min': np.min(img),\n        'img_max': np.max(img),\n        'img_mean': np.mean(img),\n        'img_std': np.std(img)}\n\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.984405Z","iopub.execute_input":"2022-12-22T16:13:38.984853Z","iopub.status.idle":"2022-12-22T16:13:38.995237Z","shell.execute_reply.started":"2022-12-22T16:13:38.984786Z","shell.execute_reply":"2022-12-22T16:13:38.994311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport tqdm \nimport pydicom\n\ntrain_image_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/train'\ntrain_image_files = glob.glob(os.path.join(train_image_path, '*', '*.dcm'))\n\nmeta_data_df = []\nfor filename in tqdm.tqdm(train_image_files):\n    try:\n        meta_data_df.append(extract_dicom_meta_data(filename))\n    except Exception as e:\n        print(e)\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:13:38.996757Z","iopub.execute_input":"2022-12-22T16:13:38.997081Z","iopub.status.idle":"2022-12-22T16:24:18.511887Z","shell.execute_reply.started":"2022-12-22T16:13:38.997052Z","shell.execute_reply":"2022-12-22T16:24:18.510544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df = pd.DataFrame.from_dict(meta_data_df)\nmeta_data_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:18.513778Z","iopub.execute_input":"2022-12-22T16:24:18.514282Z","iopub.status.idle":"2022-12-22T16:24:19.011285Z","shell.execute_reply.started":"2022-12-22T16:24:18.514232Z","shell.execute_reply":"2022-12-22T16:24:19.009925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\")\ntrain_df.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntest_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:19.013146Z","iopub.execute_input":"2022-12-22T16:24:19.015068Z","iopub.status.idle":"2022-12-22T16:24:19.037061Z","shell.execute_reply.started":"2022-12-22T16:24:19.015012Z","shell.execute_reply":"2022-12-22T16:24:19.035823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(),test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:19.050570Z","iopub.execute_input":"2022-12-22T16:24:19.051044Z","iopub.status.idle":"2022-12-22T16:24:19.067842Z","shell.execute_reply.started":"2022-12-22T16:24:19.051006Z","shell.execute_reply":"2022-12-22T16:24:19.066486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isnull()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:19.069172Z","iopub.execute_input":"2022-12-22T16:24:19.069533Z","iopub.status.idle":"2022-12-22T16:24:19.155932Z","shell.execute_reply.started":"2022-12-22T16:24:19.069499Z","shell.execute_reply":"2022-12-22T16:24:19.154769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:19.157555Z","iopub.execute_input":"2022-12-22T16:24:19.158023Z","iopub.status.idle":"2022-12-22T16:24:19.222970Z","shell.execute_reply.started":"2022-12-22T16:24:19.157982Z","shell.execute_reply":"2022-12-22T16:24:19.221471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import missingno as msno\nsns.set()\nmsno.bar(meta_data_df);\nmsno.matrix(meta_data_df);","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:19.224227Z","iopub.execute_input":"2022-12-22T16:24:19.224547Z","iopub.status.idle":"2022-12-22T16:24:23.296156Z","shell.execute_reply.started":"2022-12-22T16:24:19.224517Z","shell.execute_reply":"2022-12-22T16:24:23.294959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '../input/osic-pulmonary-fibrosis-progression/train/'\ndef show_dcm_info(file_path):\n    #print(colored(\"Filename.........:\",'yellow'),file_path)\n    #print()\n    print((\"File Path...........:\",'blue'), file_path)\n    \n    dataset = pydicom.dcmread(file_path)\n\n    pat_name = dataset.PatientName\n    display_name = pat_name.family_name + \", \" + pat_name.given_name\n    \n    print((\"Patient's name......:\",'blue'), display_name)\n    print((\"Patient id..........:\",'blue'), dataset.PatientID)\n    print((\"Patient's Sex.......:\",'blue'), dataset.PatientSex)\n    print((\"Modality............:\",'blue'), dataset.Modality)\n    print((\"Body Part Examined..:\",'blue'), dataset.BodyPartExamined)\n    \n    if 'PixelData' in dataset:\n        rows = int(dataset.Rows)\n        cols = int(dataset.Columns)\n        print((\"Image size..........:\",'blue'),\" {rows:d} x {cols:d}, {size:d} bytes\".format(\n            rows=rows, cols=cols, size=len(dataset.PixelData)))\n        if 'PixelSpacing' in dataset:\n            print((\"Pixel spacing.......:\",'blue'),dataset.PixelSpacing)\n            dataset.PixelSpacing = [1, 1]\n        plt.figure(figsize=(10, 10))\n        plt.imshow(dataset.pixel_array, cmap='gray')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.299761Z","iopub.execute_input":"2022-12-22T16:24:23.300199Z","iopub.status.idle":"2022-12-22T16:24:23.311621Z","shell.execute_reply.started":"2022-12-22T16:24:23.300159Z","shell.execute_reply":"2022-12-22T16:24:23.310262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_dcm_info(train_dir + 'ID00027637202179689871102/11.dcm')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.313767Z","iopub.execute_input":"2022-12-22T16:24:23.314190Z","iopub.status.idle":"2022-12-22T16:24:23.720419Z","shell.execute_reply.started":"2022-12-22T16:24:23.314154Z","shell.execute_reply":"2022-12-22T16:24:23.719190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.722096Z","iopub.execute_input":"2022-12-22T16:24:23.722461Z","iopub.status.idle":"2022-12-22T16:24:23.729601Z","shell.execute_reply.started":"2022-12-22T16:24:23.722426Z","shell.execute_reply":"2022-12-22T16:24:23.728165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.731469Z","iopub.execute_input":"2022-12-22T16:24:23.731861Z","iopub.status.idle":"2022-12-22T16:24:23.742571Z","shell.execute_reply.started":"2022-12-22T16:24:23.731825Z","shell.execute_reply":"2022-12-22T16:24:23.741485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.744692Z","iopub.execute_input":"2022-12-22T16:24:23.745553Z","iopub.status.idle":"2022-12-22T16:24:23.758729Z","shell.execute_reply.started":"2022-12-22T16:24:23.745500Z","shell.execute_reply":"2022-12-22T16:24:23.757689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=128","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.760410Z","iopub.execute_input":"2022-12-22T16:24:23.760964Z","iopub.status.idle":"2022-12-22T16:24:23.770236Z","shell.execute_reply.started":"2022-12-22T16:24:23.760926Z","shell.execute_reply":"2022-12-22T16:24:23.768880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\nsub_df['Patient'] = sub_df['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub_df['Weeks'] = sub_df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub_df = sub_df[['Patient','Weeks','Confidence','Patient_Week']]\nsub_df = sub_df.merge(test_df.drop('Weeks', axis=1), on=\"Patient\")\nsub_df","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:23.771631Z","iopub.execute_input":"2022-12-22T16:24:23.772123Z","iopub.status.idle":"2022-12-22T16:24:23.808837Z","shell.execute_reply.started":"2022-12-22T16:24:23.772079Z","shell.execute_reply":"2022-12-22T16:24:23.807531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['WHERE'] = 'train'\ntest_df['WHERE'] = 'val'\nsub_df['WHERE'] = 'test'\ndata = train_df.append([test_df, sub_df])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:24:59.437929Z","iopub.execute_input":"2022-12-22T16:24:59.438397Z","iopub.status.idle":"2022-12-22T16:24:59.451802Z","shell.execute_reply.started":"2022-12-22T16:24:59.438361Z","shell.execute_reply":"2022-12-22T16:24:59.450440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape, test_df.shape, sub_df.shape, data.shape)\nprint(train_df.Patient.nunique(), test_df.Patient.nunique(), sub_df.Patient.nunique(), data.Patient.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:25:34.705172Z","iopub.execute_input":"2022-12-22T16:25:34.705650Z","iopub.status.idle":"2022-12-22T16:25:34.715512Z","shell.execute_reply.started":"2022-12-22T16:25:34.705614Z","shell.execute_reply":"2022-12-22T16:25:34.714143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:26:07.955544Z","iopub.execute_input":"2022-12-22T16:26:07.956647Z","iopub.status.idle":"2022-12-22T16:26:07.966345Z","shell.execute_reply.started":"2022-12-22T16:26:07.956602Z","shell.execute_reply":"2022-12-22T16:26:07.965135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:26:18.670870Z","iopub.execute_input":"2022-12-22T16:26:18.671480Z","iopub.status.idle":"2022-12-22T16:26:18.684239Z","shell.execute_reply.started":"2022-12-22T16:26:18.671444Z","shell.execute_reply":"2022-12-22T16:26:18.683059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:26:26.210779Z","iopub.execute_input":"2022-12-22T16:26:26.211180Z","iopub.status.idle":"2022-12-22T16:26:26.225501Z","shell.execute_reply.started":"2022-12-22T16:26:26.211147Z","shell.execute_reply":"2022-12-22T16:26:26.224283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:26:34.560752Z","iopub.execute_input":"2022-12-22T16:26:34.561181Z","iopub.status.idle":"2022-12-22T16:26:34.573313Z","shell.execute_reply.started":"2022-12-22T16:26:34.561144Z","shell.execute_reply":"2022-12-22T16:26:34.572386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_height(row):\n    if row['Sex'] == 'Male':\n        return row['FVC']*90 / ((27.63 - 0.112 * row['Age'])*row[\"Percent\"])\n    else:\n        return row['FVC']*90 / ((21.78 - 0.101 * row['Age'])*row[\"Percent\"])\n    \ndata[\"Height\"] = data.apply(calculate_height,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:26:42.225411Z","iopub.execute_input":"2022-12-22T16:26:42.226409Z","iopub.status.idle":"2022-12-22T16:26:42.290857Z","shell.execute_reply.started":"2022-12-22T16:26:42.226367Z","shell.execute_reply":"2022-12-22T16:26:42.289877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:28:58.586956Z","iopub.execute_input":"2022-12-22T16:28:58.587465Z","iopub.status.idle":"2022-12-22T16:28:58.602821Z","shell.execute_reply.started":"2022-12-22T16:28:58.587426Z","shell.execute_reply":"2022-12-22T16:28:58.601348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = data.loc[data.WHERE=='train']\ntest_df = data.loc[data.WHERE=='val']\nsub_df = data.loc[data.WHERE=='test']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:29:37.549500Z","iopub.execute_input":"2022-12-22T16:29:37.549961Z","iopub.status.idle":"2022-12-22T16:29:37.561384Z","shell.execute_reply.started":"2022-12-22T16:29:37.549926Z","shell.execute_reply":"2022-12-22T16:29:37.560050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    # confidenceを2倍に\n    #sigma = 2*(y_pred[:, 2] - y_pred[:, 0])\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.05, 0.50, 0.995]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model(nh):\n    z = L.Input((nh,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:29:52.812311Z","iopub.execute_input":"2022-12-22T16:29:52.812744Z","iopub.status.idle":"2022-12-22T16:29:52.830687Z","shell.execute_reply.started":"2022-12-22T16:29:52.812699Z","shell.execute_reply":"2022-12-22T16:29:52.829262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_df['FVC'].values.astype(\"float32\")\nz = train_df[FE].values\nze = sub_df[FE].values\nnh = z.shape[1]\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:30:13.440127Z","iopub.execute_input":"2022-12-22T16:30:13.440585Z","iopub.status.idle":"2022-12-22T16:30:13.451099Z","shell.execute_reply.started":"2022-12-22T16:30:13.440546Z","shell.execute_reply":"2022-12-22T16:30:13.449761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = make_model(nh)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:30:26.735765Z","iopub.execute_input":"2022-12-22T16:30:26.736171Z","iopub.status.idle":"2022-12-22T16:30:26.796871Z","shell.execute_reply.started":"2022-12-22T16:30:26.736138Z","shell.execute_reply":"2022-12-22T16:30:26.795486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NFOLD = 3\nkf = KFold(n_splits=NFOLD)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:35:53.183884Z","iopub.execute_input":"2022-12-22T16:35:53.185248Z","iopub.status.idle":"2022-12-22T16:35:53.190589Z","shell.execute_reply.started":"2022-12-22T16:35:53.185205Z","shell.execute_reply":"2022-12-22T16:35:53.189358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncnt = 0\nEPOCHS = 800\nfor tr_idx, val_idx in kf.split(z):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    net = make_model(nh)\n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:35:54.325347Z","iopub.execute_input":"2022-12-22T16:35:54.325761Z","iopub.status.idle":"2022-12-22T16:39:04.822274Z","shell.execute_reply.started":"2022-12-22T16:35:54.325727Z","shell.execute_reply":"2022-12-22T16:39:04.821064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:04.824291Z","iopub.execute_input":"2022-12-22T16:39:04.824626Z","iopub.status.idle":"2022-12-22T16:39:04.832835Z","shell.execute_reply.started":"2022-12-22T16:39:04.824594Z","shell.execute_reply":"2022-12-22T16:39:04.831142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idxs = np.random.randint(0, y.shape[0], 100)\nplt.plot(y[idxs], label=\"ground truth\")\nplt.plot(pred[idxs, 0], label=\"q20\")\nplt.plot(pred[idxs, 1], label=\"q50\")\nplt.plot(pred[idxs, 2], label=\"q80\")\nplt.legend(loc=\"best\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:04.834636Z","iopub.execute_input":"2022-12-22T16:39:04.835055Z","iopub.status.idle":"2022-12-22T16:39:05.146486Z","shell.execute_reply.started":"2022-12-22T16:39:04.835024Z","shell.execute_reply":"2022-12-22T16:39:05.145101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(2.3*unc)\nplt.title(\"uncertainty in prediction\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:05.149252Z","iopub.execute_input":"2022-12-22T16:39:05.149609Z","iopub.status.idle":"2022-12-22T16:39:05.427994Z","shell.execute_reply.started":"2022-12-22T16:39:05.149578Z","shell.execute_reply":"2022-12-22T16:39:05.427090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['FVC1'] = 0.996*pe[:, 1]\nsub_df['Confidence1'] = 2.3*(pe[:, 2] - pe[:, 0])","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:05.429107Z","iopub.execute_input":"2022-12-22T16:39:05.429596Z","iopub.status.idle":"2022-12-22T16:39:05.434944Z","shell.execute_reply.started":"2022-12-22T16:39:05.429564Z","shell.execute_reply":"2022-12-22T16:39:05.433841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df = sub_df[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:05.436472Z","iopub.execute_input":"2022-12-22T16:39:05.436878Z","iopub.status.idle":"2022-12-22T16:39:05.448179Z","shell.execute_reply.started":"2022-12-22T16:39:05.436840Z","shell.execute_reply":"2022-12-22T16:39:05.447184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df.loc[~Sub_df.FVC1.isnull()].head(10)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:05.450479Z","iopub.execute_input":"2022-12-22T16:39:05.451485Z","iopub.status.idle":"2022-12-22T16:39:05.470271Z","shell.execute_reply.started":"2022-12-22T16:39:05.451433Z","shell.execute_reply":"2022-12-22T16:39:05.468864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df.loc[~Sub_df.FVC1.isnull(),'FVC'] = Sub_df.loc[~Sub_df.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    Sub_df['Confidence'] = sigma_opt\nelse:\n    Sub_df.loc[~Sub_df.FVC1.isnull(),'Confidence'] = Sub_df.loc[~Sub_df.FVC1.isnull(),'Confidence1']","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:39:05.471922Z","iopub.execute_input":"2022-12-22T16:39:05.472315Z","iopub.status.idle":"2022-12-22T16:39:05.483874Z","shell.execute_reply.started":"2022-12-22T16:39:05.472281Z","shell.execute_reply":"2022-12-22T16:39:05.482492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-22T16:41:14.180500Z","iopub.execute_input":"2022-12-22T16:41:14.180936Z","iopub.status.idle":"2022-12-22T16:41:14.195378Z","shell.execute_reply.started":"2022-12-22T16:41:14.180901Z","shell.execute_reply":"2022-12-22T16:41:14.193936Z"},"trusted":true},"execution_count":null,"outputs":[]}]}