{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-26T16:35:38.454856Z","iopub.execute_input":"2022-12-26T16:35:38.455701Z","iopub.status.idle":"2022-12-26T16:36:03.765267Z","shell.execute_reply.started":"2022-12-26T16:35:38.455588Z","shell.execute_reply":"2022-12-26T16:36:03.763794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport plotly.express as px\nimport plotly.graph_objs as go\n\nimport pydicom\nimport glob\nimport imageio\nfrom IPython.display import Image","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:03.766830Z","iopub.execute_input":"2022-12-26T16:36:03.767528Z","iopub.status.idle":"2022-12-26T16:36:05.296479Z","shell.execute_reply.started":"2022-12-26T16:36:03.767485Z","shell.execute_reply":"2022-12-26T16:36:05.295188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\n\nimport pydicom\nfrom skimage.transform import resize\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:05.297842Z","iopub.execute_input":"2022-12-26T16:36:05.298753Z","iopub.status.idle":"2022-12-26T16:36:06.231151Z","shell.execute_reply.started":"2022-12-26T16:36:05.298715Z","shell.execute_reply":"2022-12-26T16:36:06.230004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_df = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.234797Z","iopub.execute_input":"2022-12-26T16:36:06.235560Z","iopub.status.idle":"2022-12-26T16:36:06.276731Z","shell.execute_reply.started":"2022-12-26T16:36:06.235507Z","shell.execute_reply":"2022-12-26T16:36:06.275648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.278248Z","iopub.execute_input":"2022-12-26T16:36:06.278609Z","iopub.status.idle":"2022-12-26T16:36:06.303439Z","shell.execute_reply.started":"2022-12-26T16:36:06.278574Z","shell.execute_reply":"2022-12-26T16:36:06.302174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.305012Z","iopub.execute_input":"2022-12-26T16:36:06.305386Z","iopub.status.idle":"2022-12-26T16:36:06.336343Z","shell.execute_reply.started":"2022-12-26T16:36:06.305349Z","shell.execute_reply":"2022-12-26T16:36:06.334744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.338478Z","iopub.execute_input":"2022-12-26T16:36:06.338867Z","iopub.status.idle":"2022-12-26T16:36:06.350421Z","shell.execute_reply.started":"2022-12-26T16:36:06.338815Z","shell.execute_reply":"2022-12-26T16:36:06.348955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.352646Z","iopub.execute_input":"2022-12-26T16:36:06.353163Z","iopub.status.idle":"2022-12-26T16:36:06.369588Z","shell.execute_reply.started":"2022-12-26T16:36:06.353114Z","shell.execute_reply":"2022-12-26T16:36:06.368172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Weeks'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.371798Z","iopub.execute_input":"2022-12-26T16:36:06.372297Z","iopub.status.idle":"2022-12-26T16:36:06.757808Z","shell.execute_reply.started":"2022-12-26T16:36:06.372251Z","shell.execute_reply":"2022-12-26T16:36:06.756376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Weeks'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.762002Z","iopub.execute_input":"2022-12-26T16:36:06.762424Z","iopub.status.idle":"2022-12-26T16:36:06.777610Z","shell.execute_reply.started":"2022-12-26T16:36:06.762386Z","shell.execute_reply":"2022-12-26T16:36:06.776395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['FVC'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:06.779573Z","iopub.execute_input":"2022-12-26T16:36:06.780090Z","iopub.status.idle":"2022-12-26T16:36:07.112281Z","shell.execute_reply.started":"2022-12-26T16:36:06.780030Z","shell.execute_reply":"2022-12-26T16:36:07.110976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['FVC'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:07.114145Z","iopub.execute_input":"2022-12-26T16:36:07.114632Z","iopub.status.idle":"2022-12-26T16:36:07.128576Z","shell.execute_reply.started":"2022-12-26T16:36:07.114585Z","shell.execute_reply":"2022-12-26T16:36:07.127423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Percent'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:07.130418Z","iopub.execute_input":"2022-12-26T16:36:07.130773Z","iopub.status.idle":"2022-12-26T16:36:07.644322Z","shell.execute_reply.started":"2022-12-26T16:36:07.130741Z","shell.execute_reply":"2022-12-26T16:36:07.643080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Percent'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:07.645747Z","iopub.execute_input":"2022-12-26T16:36:07.646102Z","iopub.status.idle":"2022-12-26T16:36:07.661222Z","shell.execute_reply.started":"2022-12-26T16:36:07.646068Z","shell.execute_reply":"2022-12-26T16:36:07.659681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Age'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:07.663145Z","iopub.execute_input":"2022-12-26T16:36:07.664173Z","iopub.status.idle":"2022-12-26T16:36:08.001481Z","shell.execute_reply.started":"2022-12-26T16:36:07.664119Z","shell.execute_reply":"2022-12-26T16:36:08.000095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Age'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.003359Z","iopub.execute_input":"2022-12-26T16:36:08.003851Z","iopub.status.idle":"2022-12-26T16:36:08.017275Z","shell.execute_reply.started":"2022-12-26T16:36:08.003802Z","shell.execute_reply":"2022-12-26T16:36:08.015924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Sex'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.019220Z","iopub.execute_input":"2022-12-26T16:36:08.019757Z","iopub.status.idle":"2022-12-26T16:36:08.307478Z","shell.execute_reply.started":"2022-12-26T16:36:08.019711Z","shell.execute_reply":"2022-12-26T16:36:08.304641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Sex'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.309203Z","iopub.execute_input":"2022-12-26T16:36:08.309582Z","iopub.status.idle":"2022-12-26T16:36:08.321417Z","shell.execute_reply.started":"2022-12-26T16:36:08.309540Z","shell.execute_reply":"2022-12-26T16:36:08.320090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['SmokingStatus'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.323370Z","iopub.execute_input":"2022-12-26T16:36:08.323929Z","iopub.status.idle":"2022-12-26T16:36:08.639890Z","shell.execute_reply.started":"2022-12-26T16:36:08.323863Z","shell.execute_reply":"2022-12-26T16:36:08.638522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['SmokingStatus'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.641736Z","iopub.execute_input":"2022-12-26T16:36:08.642135Z","iopub.status.idle":"2022-12-26T16:36:08.653004Z","shell.execute_reply.started":"2022-12-26T16:36:08.642099Z","shell.execute_reply":"2022-12-26T16:36:08.652000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Weeks')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.654284Z","iopub.execute_input":"2022-12-26T16:36:08.655197Z","iopub.status.idle":"2022-12-26T16:36:08.679415Z","shell.execute_reply.started":"2022-12-26T16:36:08.655157Z","shell.execute_reply":"2022-12-26T16:36:08.678025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Weeks')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.681132Z","iopub.execute_input":"2022-12-26T16:36:08.681508Z","iopub.status.idle":"2022-12-26T16:36:08.920883Z","shell.execute_reply.started":"2022-12-26T16:36:08.681473Z","shell.execute_reply":"2022-12-26T16:36:08.919725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Percent')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.922648Z","iopub.execute_input":"2022-12-26T16:36:08.923008Z","iopub.status.idle":"2022-12-26T16:36:08.943429Z","shell.execute_reply.started":"2022-12-26T16:36:08.922974Z","shell.execute_reply":"2022-12-26T16:36:08.942070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Percent')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:08.945099Z","iopub.execute_input":"2022-12-26T16:36:08.945544Z","iopub.status.idle":"2022-12-26T16:36:09.187790Z","shell.execute_reply.started":"2022-12-26T16:36:08.945507Z","shell.execute_reply":"2022-12-26T16:36:09.186604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Age')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.189362Z","iopub.execute_input":"2022-12-26T16:36:09.189976Z","iopub.status.idle":"2022-12-26T16:36:09.206243Z","shell.execute_reply.started":"2022-12-26T16:36:09.189941Z","shell.execute_reply":"2022-12-26T16:36:09.204958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Age')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.207888Z","iopub.execute_input":"2022-12-26T16:36:09.208732Z","iopub.status.idle":"2022-12-26T16:36:09.437873Z","shell.execute_reply.started":"2022-12-26T16:36:09.208693Z","shell.execute_reply":"2022-12-26T16:36:09.436635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Sex')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.439474Z","iopub.execute_input":"2022-12-26T16:36:09.439858Z","iopub.status.idle":"2022-12-26T16:36:09.456762Z","shell.execute_reply.started":"2022-12-26T16:36:09.439820Z","shell.execute_reply":"2022-12-26T16:36:09.455384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('Sex')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.466405Z","iopub.execute_input":"2022-12-26T16:36:09.467017Z","iopub.status.idle":"2022-12-26T16:36:09.671521Z","shell.execute_reply.started":"2022-12-26T16:36:09.466973Z","shell.execute_reply":"2022-12-26T16:36:09.669936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('SmokingStatus')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.673326Z","iopub.execute_input":"2022-12-26T16:36:09.673756Z","iopub.status.idle":"2022-12-26T16:36:09.690109Z","shell.execute_reply.started":"2022-12-26T16:36:09.673719Z","shell.execute_reply":"2022-12-26T16:36:09.688714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('SmokingStatus')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.691689Z","iopub.execute_input":"2022-12-26T16:36:09.692161Z","iopub.status.idle":"2022-12-26T16:36:09.895969Z","shell.execute_reply.started":"2022-12-26T16:36:09.692113Z","shell.execute_reply":"2022-12-26T16:36:09.895024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.897162Z","iopub.execute_input":"2022-12-26T16:36:09.898104Z","iopub.status.idle":"2022-12-26T16:36:09.920559Z","shell.execute_reply.started":"2022-12-26T16:36:09.898067Z","shell.execute_reply":"2022-12-26T16:36:09.918742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.922134Z","iopub.execute_input":"2022-12-26T16:36:09.922609Z","iopub.status.idle":"2022-12-26T16:36:09.939795Z","shell.execute_reply.started":"2022-12-26T16:36:09.922575Z","shell.execute_reply":"2022-12-26T16:36:09.938437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.942005Z","iopub.execute_input":"2022-12-26T16:36:09.942545Z","iopub.status.idle":"2022-12-26T16:36:09.952303Z","shell.execute_reply.started":"2022-12-26T16:36:09.942496Z","shell.execute_reply":"2022-12-26T16:36:09.951058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.954498Z","iopub.execute_input":"2022-12-26T16:36:09.955091Z","iopub.status.idle":"2022-12-26T16:36:09.969885Z","shell.execute_reply.started":"2022-12-26T16:36:09.955043Z","shell.execute_reply":"2022-12-26T16:36:09.968963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:09.973691Z","iopub.execute_input":"2022-12-26T16:36:09.974113Z","iopub.status.idle":"2022-12-26T16:36:10.307218Z","shell.execute_reply.started":"2022-12-26T16:36:09.974073Z","shell.execute_reply":"2022-12-26T16:36:10.305738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:10.310343Z","iopub.execute_input":"2022-12-26T16:36:10.310772Z","iopub.status.idle":"2022-12-26T16:36:10.322406Z","shell.execute_reply.started":"2022-12-26T16:36:10.310735Z","shell.execute_reply":"2022-12-26T16:36:10.321449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:10.324006Z","iopub.execute_input":"2022-12-26T16:36:10.324887Z","iopub.status.idle":"2022-12-26T16:36:10.647700Z","shell.execute_reply.started":"2022-12-26T16:36:10.324839Z","shell.execute_reply":"2022-12-26T16:36:10.646419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:10.649042Z","iopub.execute_input":"2022-12-26T16:36:10.649479Z","iopub.status.idle":"2022-12-26T16:36:10.662190Z","shell.execute_reply.started":"2022-12-26T16:36:10.649433Z","shell.execute_reply":"2022-12-26T16:36:10.660817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Percent'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:10.663390Z","iopub.execute_input":"2022-12-26T16:36:10.663722Z","iopub.status.idle":"2022-12-26T16:36:10.975579Z","shell.execute_reply.started":"2022-12-26T16:36:10.663691Z","shell.execute_reply":"2022-12-26T16:36:10.974285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Percent'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:10.977266Z","iopub.execute_input":"2022-12-26T16:36:10.977646Z","iopub.status.idle":"2022-12-26T16:36:10.990888Z","shell.execute_reply.started":"2022-12-26T16:36:10.977610Z","shell.execute_reply":"2022-12-26T16:36:10.989971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:10.992258Z","iopub.execute_input":"2022-12-26T16:36:10.993180Z","iopub.status.idle":"2022-12-26T16:36:11.328246Z","shell.execute_reply.started":"2022-12-26T16:36:10.993139Z","shell.execute_reply":"2022-12-26T16:36:11.327006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:11.329825Z","iopub.execute_input":"2022-12-26T16:36:11.330238Z","iopub.status.idle":"2022-12-26T16:36:11.343390Z","shell.execute_reply.started":"2022-12-26T16:36:11.330204Z","shell.execute_reply":"2022-12-26T16:36:11.341742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:11.345317Z","iopub.execute_input":"2022-12-26T16:36:11.346273Z","iopub.status.idle":"2022-12-26T16:36:11.646724Z","shell.execute_reply.started":"2022-12-26T16:36:11.346222Z","shell.execute_reply":"2022-12-26T16:36:11.645407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:11.648015Z","iopub.execute_input":"2022-12-26T16:36:11.648344Z","iopub.status.idle":"2022-12-26T16:36:11.660028Z","shell.execute_reply.started":"2022-12-26T16:36:11.648313Z","shell.execute_reply":"2022-12-26T16:36:11.658979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:11.661595Z","iopub.execute_input":"2022-12-26T16:36:11.662574Z","iopub.status.idle":"2022-12-26T16:36:11.962654Z","shell.execute_reply.started":"2022-12-26T16:36:11.662533Z","shell.execute_reply":"2022-12-26T16:36:11.961462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:11.964338Z","iopub.execute_input":"2022-12-26T16:36:11.967936Z","iopub.status.idle":"2022-12-26T16:36:11.979063Z","shell.execute_reply.started":"2022-12-26T16:36:11.967871Z","shell.execute_reply":"2022-12-26T16:36:11.977750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Weeks')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:11.980830Z","iopub.execute_input":"2022-12-26T16:36:11.981867Z","iopub.status.idle":"2022-12-26T16:36:12.003487Z","shell.execute_reply.started":"2022-12-26T16:36:11.981819Z","shell.execute_reply":"2022-12-26T16:36:12.002283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Weeks')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:12.005404Z","iopub.execute_input":"2022-12-26T16:36:12.006182Z","iopub.status.idle":"2022-12-26T16:36:12.238801Z","shell.execute_reply.started":"2022-12-26T16:36:12.006133Z","shell.execute_reply":"2022-12-26T16:36:12.237261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Percent')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:12.240727Z","iopub.execute_input":"2022-12-26T16:36:12.241644Z","iopub.status.idle":"2022-12-26T16:36:12.263698Z","shell.execute_reply.started":"2022-12-26T16:36:12.241587Z","shell.execute_reply":"2022-12-26T16:36:12.262378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Percent')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:12.265206Z","iopub.execute_input":"2022-12-26T16:36:12.265575Z","iopub.status.idle":"2022-12-26T16:36:12.510354Z","shell.execute_reply.started":"2022-12-26T16:36:12.265537Z","shell.execute_reply":"2022-12-26T16:36:12.509003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Age')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:12.512081Z","iopub.execute_input":"2022-12-26T16:36:12.512446Z","iopub.status.idle":"2022-12-26T16:36:12.535326Z","shell.execute_reply.started":"2022-12-26T16:36:12.512413Z","shell.execute_reply":"2022-12-26T16:36:12.534016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Age')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:12.537060Z","iopub.execute_input":"2022-12-26T16:36:12.537432Z","iopub.status.idle":"2022-12-26T16:36:13.011231Z","shell.execute_reply.started":"2022-12-26T16:36:12.537399Z","shell.execute_reply":"2022-12-26T16:36:13.009825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.012887Z","iopub.execute_input":"2022-12-26T16:36:13.013285Z","iopub.status.idle":"2022-12-26T16:36:13.031299Z","shell.execute_reply.started":"2022-12-26T16:36:13.013249Z","shell.execute_reply":"2022-12-26T16:36:13.030137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.032825Z","iopub.execute_input":"2022-12-26T16:36:13.033384Z","iopub.status.idle":"2022-12-26T16:36:13.240022Z","shell.execute_reply.started":"2022-12-26T16:36:13.033336Z","shell.execute_reply":"2022-12-26T16:36:13.238832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['FVC'].agg(['count','mean','median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.242294Z","iopub.execute_input":"2022-12-26T16:36:13.242776Z","iopub.status.idle":"2022-12-26T16:36:13.262063Z","shell.execute_reply.started":"2022-12-26T16:36:13.242737Z","shell.execute_reply":"2022-12-26T16:36:13.260826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['FVC'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.263448Z","iopub.execute_input":"2022-12-26T16:36:13.263816Z","iopub.status.idle":"2022-12-26T16:36:13.511163Z","shell.execute_reply.started":"2022-12-26T16:36:13.263784Z","shell.execute_reply":"2022-12-26T16:36:13.509814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC_pred_mean'] = train_df['FVC'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.513563Z","iopub.execute_input":"2022-12-26T16:36:13.514081Z","iopub.status.idle":"2022-12-26T16:36:13.523626Z","shell.execute_reply.started":"2022-12-26T16:36:13.514031Z","shell.execute_reply":"2022-12-26T16:36:13.521762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err'] = train_df['FVC'] - train_df['FVC_pred_mean']","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.525623Z","iopub.execute_input":"2022-12-26T16:36:13.526190Z","iopub.status.idle":"2022-12-26T16:36:13.542561Z","shell.execute_reply.started":"2022-12-26T16:36:13.526146Z","shell.execute_reply":"2022-12-26T16:36:13.540801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs'] = abs(train_df['err'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.544717Z","iopub.execute_input":"2022-12-26T16:36:13.545699Z","iopub.status.idle":"2022-12-26T16:36:13.558440Z","shell.execute_reply.started":"2022-12-26T16:36:13.545645Z","shell.execute_reply":"2022-12-26T16:36:13.557257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.560315Z","iopub.execute_input":"2022-12-26T16:36:13.560972Z","iopub.status.idle":"2022-12-26T16:36:13.571734Z","shell.execute_reply.started":"2022-12-26T16:36:13.560928Z","shell.execute_reply":"2022-12-26T16:36:13.570791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs_pct'] = train_df['err_abs'] / train_df['FVC']","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.573287Z","iopub.execute_input":"2022-12-26T16:36:13.574231Z","iopub.status.idle":"2022-12-26T16:36:13.584390Z","shell.execute_reply.started":"2022-12-26T16:36:13.574195Z","shell.execute_reply":"2022-12-26T16:36:13.583347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['err_abs_pct'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.586311Z","iopub.execute_input":"2022-12-26T16:36:13.587010Z","iopub.status.idle":"2022-12-26T16:36:13.600968Z","shell.execute_reply.started":"2022-12-26T16:36:13.586972Z","shell.execute_reply":"2022-12-26T16:36:13.599476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC_pred_median'] = train_df['FVC'].median()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.603274Z","iopub.execute_input":"2022-12-26T16:36:13.604237Z","iopub.status.idle":"2022-12-26T16:36:13.611254Z","shell.execute_reply.started":"2022-12-26T16:36:13.604181Z","shell.execute_reply":"2022-12-26T16:36:13.610175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.613346Z","iopub.execute_input":"2022-12-26T16:36:13.614123Z","iopub.status.idle":"2022-12-26T16:36:13.644679Z","shell.execute_reply.started":"2022-12-26T16:36:13.614084Z","shell.execute_reply":"2022-12-26T16:36:13.643648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.646240Z","iopub.execute_input":"2022-12-26T16:36:13.646790Z","iopub.status.idle":"2022-12-26T16:36:13.696246Z","shell.execute_reply.started":"2022-12-26T16:36:13.646755Z","shell.execute_reply":"2022-12-26T16:36:13.695017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_error(train_df['FVC'],train_df['FVC_pred_mean'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.697546Z","iopub.execute_input":"2022-12-26T16:36:13.698195Z","iopub.status.idle":"2022-12-26T16:36:13.708350Z","shell.execute_reply.started":"2022-12-26T16:36:13.698153Z","shell.execute_reply":"2022-12-26T16:36:13.706991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_percentage_error(train_df['FVC'],train_df['FVC_pred_mean'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.709658Z","iopub.execute_input":"2022-12-26T16:36:13.710069Z","iopub.status.idle":"2022-12-26T16:36:13.722433Z","shell.execute_reply.started":"2022-12-26T16:36:13.710027Z","shell.execute_reply":"2022-12-26T16:36:13.721177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_error(train_df['FVC'],train_df['FVC_pred_median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.724155Z","iopub.execute_input":"2022-12-26T16:36:13.725204Z","iopub.status.idle":"2022-12-26T16:36:13.733682Z","shell.execute_reply.started":"2022-12-26T16:36:13.725156Z","shell.execute_reply":"2022-12-26T16:36:13.732795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_absolute_percentage_error(train_df['FVC'],train_df['FVC_pred_median'])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.734853Z","iopub.execute_input":"2022-12-26T16:36:13.735404Z","iopub.status.idle":"2022-12-26T16:36:13.748093Z","shell.execute_reply.started":"2022-12-26T16:36:13.735355Z","shell.execute_reply":"2022-12-26T16:36:13.747251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.749345Z","iopub.execute_input":"2022-12-26T16:36:13.749974Z","iopub.status.idle":"2022-12-26T16:36:13.781416Z","shell.execute_reply.started":"2022-12-26T16:36:13.749927Z","shell.execute_reply":"2022-12-26T16:36:13.780005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates = train_df[train_df.duplicated(subset = ['Patient', 'Weeks'], keep = False)]\nduplicates","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.783369Z","iopub.execute_input":"2022-12-26T16:36:13.784463Z","iopub.status.idle":"2022-12-26T16:36:13.811980Z","shell.execute_reply.started":"2022-12-26T16:36:13.784423Z","shell.execute_reply":"2022-12-26T16:36:13.810737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\n\ndef extract_dicom_meta_data(filename: str) -> Dict:\n    # Load image\n    \n    image_data = pydicom.read_file(filename)\n    img=np.array(image_data.pixel_array).flatten()\n    row = {\n        'Patient': image_data.PatientID,\n        'body_part_examined': image_data.BodyPartExamined,\n        'image_position_patient': image_data.ImagePositionPatient,\n        'image_orientation_patient': image_data.ImageOrientationPatient,\n        'photometric_interpretation': image_data.PhotometricInterpretation,\n        'rows': image_data.Rows,\n        'columns': image_data.Columns,\n        'pixel_spacing': image_data.PixelSpacing,\n        'window_center': image_data.WindowCenter,\n        'window_width': image_data.WindowWidth,\n        'modality': image_data.Modality,\n        'StudyInstanceUID': image_data.StudyInstanceUID,\n        'SeriesInstanceUID': image_data.StudyInstanceUID,\n        'StudyID': image_data.StudyInstanceUID, \n        'SamplesPerPixel': image_data.SamplesPerPixel,\n        'BitsAllocated': image_data.BitsAllocated,\n        'BitsStored': image_data.BitsStored,\n        'HighBit': image_data.HighBit,\n        'PixelRepresentation': image_data.PixelRepresentation,\n        'RescaleIntercept': image_data.RescaleIntercept,\n        'RescaleSlope': image_data.RescaleSlope,\n        'img_min': np.min(img),\n        'img_max': np.max(img),\n        'img_mean': np.mean(img),\n        'img_std': np.std(img)}\n\n    return row","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.813480Z","iopub.execute_input":"2022-12-26T16:36:13.813837Z","iopub.status.idle":"2022-12-26T16:36:13.825704Z","shell.execute_reply.started":"2022-12-26T16:36:13.813804Z","shell.execute_reply":"2022-12-26T16:36:13.823998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport tqdm \nimport pydicom\n\ntrain_image_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/train'\ntrain_image_files = glob.glob(os.path.join(train_image_path, '*', '*.dcm'))\n\nmeta_data_df = []\nfor filename in tqdm.tqdm(train_image_files):\n    try:\n        meta_data_df.append(extract_dicom_meta_data(filename))\n    except Exception as e:\n        print(e)\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:36:13.828726Z","iopub.execute_input":"2022-12-26T16:36:13.829151Z","iopub.status.idle":"2022-12-26T16:49:35.368919Z","shell.execute_reply.started":"2022-12-26T16:36:13.829115Z","shell.execute_reply":"2022-12-26T16:49:35.366073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df = pd.DataFrame.from_dict(meta_data_df)\nmeta_data_df","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:35.370944Z","iopub.execute_input":"2022-12-26T16:49:35.373274Z","iopub.status.idle":"2022-12-26T16:49:35.901267Z","shell.execute_reply.started":"2022-12-26T16:49:35.373223Z","shell.execute_reply":"2022-12-26T16:49:35.900113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\")\ntrain_df.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntest_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv\")\nsub_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\nsub_df['Patient'] = sub_df['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub_df['Weeks'] = sub_df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub_df = sub_df[['Patient','Weeks','Confidence','Patient_Week']]\nsub_df = sub_df.merge(sub_df.drop('Weeks', axis=1), on=\"Patient\")","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:35.903140Z","iopub.execute_input":"2022-12-26T16:49:35.903492Z","iopub.status.idle":"2022-12-26T16:49:35.974848Z","shell.execute_reply.started":"2022-12-26T16:49:35.903459Z","shell.execute_reply":"2022-12-26T16:49:35.973656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(),test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:35.977071Z","iopub.execute_input":"2022-12-26T16:49:35.977565Z","iopub.status.idle":"2022-12-26T16:49:35.995697Z","shell.execute_reply.started":"2022-12-26T16:49:35.977518Z","shell.execute_reply":"2022-12-26T16:49:35.994379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pixel_array(dataset, figsize=(5,5)):\n    plt.figure(figsize=figsize)\n    plt.grid(False)\n    plt.imshow(dataset.pixel_array, cmap='gray') # cmap=plt.cm.bone)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:35.998230Z","iopub.execute_input":"2022-12-26T16:49:35.999067Z","iopub.status.idle":"2022-12-26T16:49:36.008592Z","shell.execute_reply.started":"2022-12-26T16:49:35.999016Z","shell.execute_reply":"2022-12-26T16:49:36.007148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imdir = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00123637202217151272140\"\nprint(\"total images for patient ID00123637202217151272140: \", len(os.listdir(imdir)))\nfig=plt.figure(figsize=(12, 12))\ncolumns = 4\nrows = 5\nimglist = os.listdir(imdir)\nfor i in range(1, columns*rows +1):\n    filename = imdir + \"/\" + str(i) + \".dcm\"\n    ds = pydicom.dcmread(filename)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(ds.pixel_array, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:36.010682Z","iopub.execute_input":"2022-12-26T16:49:36.011216Z","iopub.status.idle":"2022-12-26T16:49:38.534136Z","shell.execute_reply.started":"2022-12-26T16:49:36.011166Z","shell.execute_reply":"2022-12-26T16:49:38.533040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imdir = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00123637202217151272140\"\nprint(\"total images for patient ID00123637202217151272140: \", len(os.listdir(imdir)))\nfig=plt.figure(figsize=(12, 12))\ncolumns = 4\nrows = 5\nimglist = os.listdir(imdir)\nfor i in range(1, columns*rows +1):\n    filename = imdir + \"/\" + str(i) + \".dcm\"\n    ds = pydicom.dcmread(filename)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(ds.pixel_array, cmap='jet')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:38.535715Z","iopub.execute_input":"2022-12-26T16:49:38.536046Z","iopub.status.idle":"2022-12-26T16:49:41.077692Z","shell.execute_reply.started":"2022-12-26T16:49:38.536015Z","shell.execute_reply":"2022-12-26T16:49:41.076312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isnull()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:41.088484Z","iopub.execute_input":"2022-12-26T16:49:41.089191Z","iopub.status.idle":"2022-12-26T16:49:41.179145Z","shell.execute_reply.started":"2022-12-26T16:49:41.089149Z","shell.execute_reply":"2022-12-26T16:49:41.178233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:41.180437Z","iopub.execute_input":"2022-12-26T16:49:41.181003Z","iopub.status.idle":"2022-12-26T16:49:41.246654Z","shell.execute_reply.started":"2022-12-26T16:49:41.180959Z","shell.execute_reply":"2022-12-26T16:49:41.245323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import missingno as msno\nsns.set()\nmsno.bar(meta_data_df);\nmsno.matrix(meta_data_df);","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:41.248976Z","iopub.execute_input":"2022-12-26T16:49:41.249404Z","iopub.status.idle":"2022-12-26T16:49:45.304676Z","shell.execute_reply.started":"2022-12-26T16:49:41.249368Z","shell.execute_reply":"2022-12-26T16:49:45.303661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:45.306398Z","iopub.execute_input":"2022-12-26T16:49:45.306989Z","iopub.status.idle":"2022-12-26T16:49:45.330125Z","shell.execute_reply.started":"2022-12-26T16:49:45.306943Z","shell.execute_reply":"2022-12-26T16:49:45.328664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:45.331731Z","iopub.execute_input":"2022-12-26T16:49:45.332233Z","iopub.status.idle":"2022-12-26T16:49:52.667197Z","shell.execute_reply.started":"2022-12-26T16:49:45.332188Z","shell.execute_reply":"2022-12-26T16:49:52.665858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.668816Z","iopub.execute_input":"2022-12-26T16:49:52.669756Z","iopub.status.idle":"2022-12-26T16:49:52.678265Z","shell.execute_reply.started":"2022-12-26T16:49:52.669716Z","shell.execute_reply":"2022-12-26T16:49:52.677037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=128","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.679715Z","iopub.execute_input":"2022-12-26T16:49:52.680473Z","iopub.status.idle":"2022-12-26T16:49:52.707649Z","shell.execute_reply.started":"2022-12-26T16:49:52.680431Z","shell.execute_reply":"2022-12-26T16:49:52.706025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\nsub_df['Patient'] = sub_df['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub_df['Weeks'] = sub_df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub_df = sub_df[['Patient','Weeks','Confidence','Patient_Week']]\nsub_df = sub_df.merge(test_df.drop('Weeks', axis=1), on=\"Patient\")\nsub_df","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.710012Z","iopub.execute_input":"2022-12-26T16:49:52.710904Z","iopub.status.idle":"2022-12-26T16:49:52.755756Z","shell.execute_reply.started":"2022-12-26T16:49:52.710851Z","shell.execute_reply":"2022-12-26T16:49:52.754367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['WHERE'] = 'train'\ntest_df['WHERE'] = 'val'\nsub_df['WHERE'] = 'test'\ndata = train_df.append([test_df, sub_df])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.758041Z","iopub.execute_input":"2022-12-26T16:49:52.758963Z","iopub.status.idle":"2022-12-26T16:49:52.773908Z","shell.execute_reply.started":"2022-12-26T16:49:52.758893Z","shell.execute_reply":"2022-12-26T16:49:52.772005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.shape, test_df.shape, sub_df.shape, data.shape)\nprint(train_df.Patient.nunique(), test_df.Patient.nunique(), sub_df.Patient.nunique(), data.Patient.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.776484Z","iopub.execute_input":"2022-12-26T16:49:52.777363Z","iopub.status.idle":"2022-12-26T16:49:52.788201Z","shell.execute_reply.started":"2022-12-26T16:49:52.777310Z","shell.execute_reply":"2022-12-26T16:49:52.786746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.790485Z","iopub.execute_input":"2022-12-26T16:49:52.791277Z","iopub.status.idle":"2022-12-26T16:49:52.806992Z","shell.execute_reply.started":"2022-12-26T16:49:52.791227Z","shell.execute_reply":"2022-12-26T16:49:52.805516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.808426Z","iopub.execute_input":"2022-12-26T16:49:52.808779Z","iopub.status.idle":"2022-12-26T16:49:52.827892Z","shell.execute_reply.started":"2022-12-26T16:49:52.808748Z","shell.execute_reply":"2022-12-26T16:49:52.826789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.829209Z","iopub.execute_input":"2022-12-26T16:49:52.830377Z","iopub.status.idle":"2022-12-26T16:49:52.848619Z","shell.execute_reply.started":"2022-12-26T16:49:52.830143Z","shell.execute_reply":"2022-12-26T16:49:52.846980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.850471Z","iopub.execute_input":"2022-12-26T16:49:52.850805Z","iopub.status.idle":"2022-12-26T16:49:52.865953Z","shell.execute_reply.started":"2022-12-26T16:49:52.850775Z","shell.execute_reply":"2022-12-26T16:49:52.864738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_height(row):\n    if row['Sex'] == 'Male':\n        return row['FVC']*90 / ((27.63 - 0.112 * row['Age'])*row[\"Percent\"])\n    else:\n        return row['FVC']*90 / ((21.78 - 0.101 * row['Age'])*row[\"Percent\"])\n    \ndata[\"Height\"] = data.apply(calculate_height,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.867444Z","iopub.execute_input":"2022-12-26T16:49:52.867774Z","iopub.status.idle":"2022-12-26T16:49:52.940976Z","shell.execute_reply.started":"2022-12-26T16:49:52.867744Z","shell.execute_reply":"2022-12-26T16:49:52.939735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.944856Z","iopub.execute_input":"2022-12-26T16:49:52.945285Z","iopub.status.idle":"2022-12-26T16:49:52.962503Z","shell.execute_reply.started":"2022-12-26T16:49:52.945249Z","shell.execute_reply":"2022-12-26T16:49:52.961022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = data.loc[data.WHERE=='train']\ntest_df = data.loc[data.WHERE=='val']\nsub_df = data.loc[data.WHERE=='test']","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.964646Z","iopub.execute_input":"2022-12-26T16:49:52.965074Z","iopub.status.idle":"2022-12-26T16:49:52.976862Z","shell.execute_reply.started":"2022-12-26T16:49:52.965038Z","shell.execute_reply":"2022-12-26T16:49:52.975449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    # confidenceを2倍に\n    #sigma = 2*(y_pred[:, 2] - y_pred[:, 0])\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.05, 0.50, 0.995]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model(nh):\n    z = L.Input((nh,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:52.979142Z","iopub.execute_input":"2022-12-26T16:49:52.979573Z","iopub.status.idle":"2022-12-26T16:49:53.032695Z","shell.execute_reply.started":"2022-12-26T16:49:52.979531Z","shell.execute_reply":"2022-12-26T16:49:53.031089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_df['FVC'].values.astype(\"float32\")\nz = train_df[FE].values\nze = sub_df[FE].values\nnh = z.shape[1]\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:53.034662Z","iopub.execute_input":"2022-12-26T16:49:53.035213Z","iopub.status.idle":"2022-12-26T16:49:53.051053Z","shell.execute_reply.started":"2022-12-26T16:49:53.035166Z","shell.execute_reply":"2022-12-26T16:49:53.049996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = make_model(nh)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:49:53.052374Z","iopub.execute_input":"2022-12-26T16:49:53.053288Z","iopub.status.idle":"2022-12-26T16:49:53.197327Z","shell.execute_reply.started":"2022-12-26T16:49:53.053252Z","shell.execute_reply":"2022-12-26T16:49:53.196014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NFOLD = 3\nkf = KFold(n_splits=NFOLD)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:51:05.166720Z","iopub.execute_input":"2022-12-26T16:51:05.167471Z","iopub.status.idle":"2022-12-26T16:51:05.173123Z","shell.execute_reply.started":"2022-12-26T16:51:05.167433Z","shell.execute_reply":"2022-12-26T16:51:05.171414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncnt = 0\nEPOCHS = 800\nfor tr_idx, val_idx in kf.split(z):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    net = make_model(nh)\n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:51:07.175629Z","iopub.execute_input":"2022-12-26T16:51:07.176367Z","iopub.status.idle":"2022-12-26T16:54:01.554049Z","shell.execute_reply.started":"2022-12-26T16:51:07.176329Z","shell.execute_reply":"2022-12-26T16:54:01.552649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:01.556221Z","iopub.execute_input":"2022-12-26T16:54:01.556557Z","iopub.status.idle":"2022-12-26T16:54:01.564167Z","shell.execute_reply.started":"2022-12-26T16:54:01.556526Z","shell.execute_reply":"2022-12-26T16:54:01.563265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idxs = np.random.randint(0, y.shape[0], 100)\nplt.plot(y[idxs], label=\"ground truth\")\nplt.plot(pred[idxs, 0], label=\"q20\")\nplt.plot(pred[idxs, 1], label=\"q50\")\nplt.plot(pred[idxs, 2], label=\"q80\")\nplt.legend(loc=\"best\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:01.565402Z","iopub.execute_input":"2022-12-26T16:54:01.565954Z","iopub.status.idle":"2022-12-26T16:54:01.888776Z","shell.execute_reply.started":"2022-12-26T16:54:01.565896Z","shell.execute_reply":"2022-12-26T16:54:01.887416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(2.3*unc)\nplt.title(\"uncertainty in prediction\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:01.892170Z","iopub.execute_input":"2022-12-26T16:54:01.892675Z","iopub.status.idle":"2022-12-26T16:54:02.156734Z","shell.execute_reply.started":"2022-12-26T16:54:01.892615Z","shell.execute_reply":"2022-12-26T16:54:02.155808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['FVC1'] = 0.996*pe[:, 1]\nsub_df['Confidence1'] = 2.3*(pe[:, 2] - pe[:, 0])","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:02.158277Z","iopub.execute_input":"2022-12-26T16:54:02.158624Z","iopub.status.idle":"2022-12-26T16:54:02.165593Z","shell.execute_reply.started":"2022-12-26T16:54:02.158592Z","shell.execute_reply":"2022-12-26T16:54:02.164410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df = sub_df[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:02.166865Z","iopub.execute_input":"2022-12-26T16:54:02.167219Z","iopub.status.idle":"2022-12-26T16:54:02.180101Z","shell.execute_reply.started":"2022-12-26T16:54:02.167187Z","shell.execute_reply":"2022-12-26T16:54:02.178498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df.loc[~Sub_df.FVC1.isnull()].head(10)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:02.181953Z","iopub.execute_input":"2022-12-26T16:54:02.182507Z","iopub.status.idle":"2022-12-26T16:54:02.203537Z","shell.execute_reply.started":"2022-12-26T16:54:02.182452Z","shell.execute_reply":"2022-12-26T16:54:02.202361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df.loc[~Sub_df.FVC1.isnull(),'FVC'] = Sub_df.loc[~Sub_df.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    Sub_df['Confidence'] = sigma_opt\nelse:\n    Sub_df.loc[~Sub_df.FVC1.isnull(),'Confidence'] = Sub_df.loc[~Sub_df.FVC1.isnull(),'Confidence1']","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:02.205054Z","iopub.execute_input":"2022-12-26T16:54:02.205868Z","iopub.status.idle":"2022-12-26T16:54:02.222423Z","shell.execute_reply.started":"2022-12-26T16:54:02.205820Z","shell.execute_reply":"2022-12-26T16:54:02.221371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Sub_df[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-26T16:54:02.223841Z","iopub.execute_input":"2022-12-26T16:54:02.224258Z","iopub.status.idle":"2022-12-26T16:54:02.243001Z","shell.execute_reply.started":"2022-12-26T16:54:02.224224Z","shell.execute_reply":"2022-12-26T16:54:02.241618Z"},"trusted":true},"execution_count":null,"outputs":[]}]}