{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-14T06:31:57.168065Z","iopub.execute_input":"2023-01-14T06:31:57.168602Z","iopub.status.idle":"2023-01-14T06:32:07.354253Z","shell.execute_reply.started":"2023-01-14T06:31:57.168503Z","shell.execute_reply":"2023-01-14T06:32:07.353093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directory = '/kaggle/input/osic-pulmonary-fibrosis-progression'\ntrain_df = pd.read_csv(directory + '/train.csv')\ntest_df = pd.read_csv(directory + '/test.csv')\n\nIMAGE_PATH = \"../input/osic-pulmonary-fibrosis-progressiont/\"","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:29.409727Z","iopub.execute_input":"2023-01-14T06:32:29.410247Z","iopub.status.idle":"2023-01-14T06:32:29.447812Z","shell.execute_reply.started":"2023-01-14T06:32:29.410197Z","shell.execute_reply":"2023-01-14T06:32:29.446607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:32.129247Z","iopub.execute_input":"2023-01-14T06:32:32.129801Z","iopub.status.idle":"2023-01-14T06:32:32.160565Z","shell.execute_reply.started":"2023-01-14T06:32:32.129756Z","shell.execute_reply":"2023-01-14T06:32:32.159390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:34.979825Z","iopub.execute_input":"2023-01-14T06:32:34.980266Z","iopub.status.idle":"2023-01-14T06:32:34.993052Z","shell.execute_reply.started":"2023-01-14T06:32:34.980228Z","shell.execute_reply":"2023-01-14T06:32:34.992040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:37.235430Z","iopub.execute_input":"2023-01-14T06:32:37.236189Z","iopub.status.idle":"2023-01-14T06:32:37.256956Z","shell.execute_reply.started":"2023-01-14T06:32:37.236148Z","shell.execute_reply":"2023-01-14T06:32:37.255645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Patient'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:39.699750Z","iopub.execute_input":"2023-01-14T06:32:39.701003Z","iopub.status.idle":"2023-01-14T06:32:39.708618Z","shell.execute_reply.started":"2023-01-14T06:32:39.700948Z","shell.execute_reply":"2023-01-14T06:32:39.707553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().mean()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:42.022934Z","iopub.execute_input":"2023-01-14T06:32:42.023340Z","iopub.status.idle":"2023-01-14T06:32:42.034078Z","shell.execute_reply.started":"2023-01-14T06:32:42.023295Z","shell.execute_reply":"2023-01-14T06:32:42.032896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:43.942035Z","iopub.execute_input":"2023-01-14T06:32:43.942467Z","iopub.status.idle":"2023-01-14T06:32:43.963741Z","shell.execute_reply.started":"2023-01-14T06:32:43.942431Z","shell.execute_reply":"2023-01-14T06:32:43.962487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nlen(list(Path(directory+'/train/').rglob(\"*\")))","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:32:58.429739Z","iopub.execute_input":"2023-01-14T06:32:58.430167Z","iopub.status.idle":"2023-01-14T06:32:58.987809Z","shell.execute_reply.started":"2023-01-14T06:32:58.430126Z","shell.execute_reply":"2023-01-14T06:32:58.986632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nlen(list(Path(directory+'/test/').rglob(\"*\")))","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:01.121231Z","iopub.execute_input":"2023-01-14T06:33:01.121702Z","iopub.status.idle":"2023-01-14T06:33:01.143216Z","shell.execute_reply.started":"2023-01-14T06:33:01.121662Z","shell.execute_reply":"2023-01-14T06:33:01.142189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:03.020026Z","iopub.execute_input":"2023-01-14T06:33:03.020465Z","iopub.status.idle":"2023-01-14T06:33:03.025742Z","shell.execute_reply.started":"2023-01-14T06:33:03.020428Z","shell.execute_reply":"2023-01-14T06:33:03.024633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:05.385354Z","iopub.execute_input":"2023-01-14T06:33:05.386124Z","iopub.status.idle":"2023-01-14T06:33:05.396282Z","shell.execute_reply.started":"2023-01-14T06:33:05.386082Z","shell.execute_reply":"2023-01-14T06:33:05.395235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Weeks'].hist(bins=40)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:08.296843Z","iopub.execute_input":"2023-01-14T06:33:08.298119Z","iopub.status.idle":"2023-01-14T06:33:08.640628Z","shell.execute_reply.started":"2023-01-14T06:33:08.298053Z","shell.execute_reply":"2023-01-14T06:33:08.639392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['First_Week'] = 0\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:07:38.475843Z","iopub.execute_input":"2023-01-14T07:07:38.477195Z","iopub.status.idle":"2023-01-14T07:07:38.500854Z","shell.execute_reply.started":"2023-01-14T07:07:38.477144Z","shell.execute_reply":"2023-01-14T07:07:38.499561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition = (train_df['Weeks'] <= 1)\ntrain_df.loc[condition, 'First_Week'] = 'One Week'\n\ncondition = (train_df['Weeks'] > 1)\ntrain_df.loc[condition, 'First_Week'] = 'More Week'\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:09:01.090228Z","iopub.execute_input":"2023-01-14T07:09:01.091544Z","iopub.status.idle":"2023-01-14T07:09:01.118511Z","shell.execute_reply.started":"2023-01-14T07:09:01.091496Z","shell.execute_reply":"2023-01-14T07:09:01.117529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['First_Week'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:09:18.388760Z","iopub.execute_input":"2023-01-14T07:09:18.389199Z","iopub.status.idle":"2023-01-14T07:09:18.398916Z","shell.execute_reply.started":"2023-01-14T07:09:18.389167Z","shell.execute_reply":"2023-01-14T07:09:18.398078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates = train_df[train_df.duplicated(subset = ['Patient', 'Weeks'], keep = False)]\nduplicates","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:09:27.616614Z","iopub.execute_input":"2023-01-14T07:09:27.617170Z","iopub.status.idle":"2023-01-14T07:09:27.643433Z","shell.execute_reply.started":"2023-01-14T07:09:27.617119Z","shell.execute_reply":"2023-01-14T07:09:27.642108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:11.348639Z","iopub.execute_input":"2023-01-14T06:33:11.349167Z","iopub.status.idle":"2023-01-14T06:33:11.363685Z","shell.execute_reply.started":"2023-01-14T06:33:11.349119Z","shell.execute_reply":"2023-01-14T06:33:11.362163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Age'].hist(bins=40)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:13.861198Z","iopub.execute_input":"2023-01-14T06:33:13.861675Z","iopub.status.idle":"2023-01-14T06:33:14.160826Z","shell.execute_reply.started":"2023-01-14T06:33:13.861636Z","shell.execute_reply":"2023-01-14T06:33:14.159705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:16.129412Z","iopub.execute_input":"2023-01-14T06:33:16.129815Z","iopub.status.idle":"2023-01-14T06:33:16.139062Z","shell.execute_reply.started":"2023-01-14T06:33:16.129783Z","shell.execute_reply":"2023-01-14T06:33:16.138038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Sex'].hist(bins=10)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:18.302730Z","iopub.execute_input":"2023-01-14T06:33:18.303126Z","iopub.status.idle":"2023-01-14T06:33:18.548582Z","shell.execute_reply.started":"2023-01-14T06:33:18.303095Z","shell.execute_reply":"2023-01-14T06:33:18.547393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:20.807404Z","iopub.execute_input":"2023-01-14T06:33:20.807842Z","iopub.status.idle":"2023-01-14T06:33:20.818990Z","shell.execute_reply.started":"2023-01-14T06:33:20.807805Z","shell.execute_reply":"2023-01-14T06:33:20.817708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['SmokingStatus'].hist(bins=10)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:23.024681Z","iopub.execute_input":"2023-01-14T06:33:23.025124Z","iopub.status.idle":"2023-01-14T06:33:23.236233Z","shell.execute_reply.started":"2023-01-14T06:33:23.025084Z","shell.execute_reply":"2023-01-14T06:33:23.234912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:26.741873Z","iopub.execute_input":"2023-01-14T06:33:26.742347Z","iopub.status.idle":"2023-01-14T06:33:26.752610Z","shell.execute_reply.started":"2023-01-14T06:33:26.742285Z","shell.execute_reply":"2023-01-14T06:33:26.751343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['FVC'].hist(bins=80)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:28.734128Z","iopub.execute_input":"2023-01-14T06:33:28.734548Z","iopub.status.idle":"2023-01-14T06:33:29.206879Z","shell.execute_reply.started":"2023-01-14T06:33:28.734515Z","shell.execute_reply":"2023-01-14T06:33:29.205683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.plot(train_df[\"Weeks\"], train_df[\"FVC\"], \"o\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:31.615614Z","iopub.execute_input":"2023-01-14T06:33:31.616008Z","iopub.status.idle":"2023-01-14T06:33:31.810791Z","shell.execute_reply.started":"2023-01-14T06:33:31.615978Z","shell.execute_reply":"2023-01-14T06:33:31.809785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['Age'].median().plot()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:34.224475Z","iopub.execute_input":"2023-01-14T06:33:34.224900Z","iopub.status.idle":"2023-01-14T06:33:34.454438Z","shell.execute_reply.started":"2023-01-14T06:33:34.224865Z","shell.execute_reply":"2023-01-14T06:33:34.453069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['Age'].hist(bins=10,histtype='step')","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:36.814798Z","iopub.execute_input":"2023-01-14T06:33:36.815228Z","iopub.status.idle":"2023-01-14T06:33:37.040766Z","shell.execute_reply.started":"2023-01-14T06:33:36.815197Z","shell.execute_reply":"2023-01-14T06:33:37.039469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('Sex')['Age'].hist(bins=10,histtype='step')","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:39.636588Z","iopub.execute_input":"2023-01-14T06:33:39.637031Z","iopub.status.idle":"2023-01-14T06:33:39.869016Z","shell.execute_reply.started":"2023-01-14T06:33:39.636996Z","shell.execute_reply":"2023-01-14T06:33:39.868155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby('SmokingStatus')['Percent'].agg(['max','min']).plot()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:33:43.095055Z","iopub.execute_input":"2023-01-14T06:33:43.095634Z","iopub.status.idle":"2023-01-14T06:33:43.298654Z","shell.execute_reply.started":"2023-01-14T06:33:43.095586Z","shell.execute_reply":"2023-01-14T06:33:43.297719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\n\ndef extract_dicom_meta_data(filename: str) -> Dict:\n    # Load image\n    \n    image_data = pydicom.read_file(filename)\n    img=np.array(image_data.pixel_array).flatten()\n    row = {\n        'Patient': image_data.PatientID,\n        'body_part_examined': image_data.BodyPartExamined,\n        'image_position_patient': image_data.ImagePositionPatient,\n        'image_orientation_patient': image_data.ImageOrientationPatient,\n        'photometric_interpretation': image_data.PhotometricInterpretation,\n        'rows': image_data.Rows,\n        'columns': image_data.Columns,\n        'pixel_spacing': image_data.PixelSpacing,\n        'window_center': image_data.WindowCenter,\n        'window_width': image_data.WindowWidth,\n        'modality': image_data.Modality,\n        'StudyInstanceUID': image_data.StudyInstanceUID,\n        'SeriesInstanceUID': image_data.StudyInstanceUID,\n        'StudyID': image_data.StudyInstanceUID, \n        'SamplesPerPixel': image_data.SamplesPerPixel,\n        'BitsAllocated': image_data.BitsAllocated,\n        'BitsStored': image_data.BitsStored,\n        'HighBit': image_data.HighBit,\n        'PixelRepresentation': image_data.PixelRepresentation,\n        'RescaleIntercept': image_data.RescaleIntercept,\n        'RescaleSlope': image_data.RescaleSlope,\n        'img_min': np.min(img),\n        'img_max': np.max(img),\n        'img_mean': np.mean(img),\n        'img_std': np.std(img)}\n\n    return row","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:34:01.286892Z","iopub.execute_input":"2023-01-14T06:34:01.287281Z","iopub.status.idle":"2023-01-14T06:34:01.297702Z","shell.execute_reply.started":"2023-01-14T06:34:01.287249Z","shell.execute_reply":"2023-01-14T06:34:01.296443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport tqdm \nimport pydicom\n\ntrain_image_path = '/kaggle/input/osic-pulmonary-fibrosis-progression/train'\ntrain_image_files = glob.glob(os.path.join(train_image_path, '*', '*.dcm'))\n\nmeta_data_df = []\nfor filename in tqdm.tqdm(train_image_files):\n    try:\n        meta_data_df.append(extract_dicom_meta_data(filename))\n    except Exception as e:\n        print(e)\n        continue","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:34:09.753666Z","iopub.execute_input":"2023-01-14T06:34:09.754070Z","iopub.status.idle":"2023-01-14T06:44:22.697176Z","shell.execute_reply.started":"2023-01-14T06:34:09.754038Z","shell.execute_reply":"2023-01-14T06:44:22.695601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df = pd.DataFrame.from_dict(meta_data_df)\nmeta_data_df","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:18.092918Z","iopub.execute_input":"2023-01-14T06:49:18.094158Z","iopub.status.idle":"2023-01-14T06:49:18.605606Z","shell.execute_reply.started":"2023-01-14T06:49:18.094098Z","shell.execute_reply":"2023-01-14T06:49:18.604329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\")\ntrain_df.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntest_df = pd.read_csv(f\"/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:16:02.285913Z","iopub.execute_input":"2023-01-14T07:16:02.286434Z","iopub.status.idle":"2023-01-14T07:16:02.318713Z","shell.execute_reply.started":"2023-01-14T07:16:02.286396Z","shell.execute_reply":"2023-01-14T07:16:02.317807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(),test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:16:12.576510Z","iopub.execute_input":"2023-01-14T07:16:12.576972Z","iopub.status.idle":"2023-01-14T07:16:12.591416Z","shell.execute_reply.started":"2023-01-14T07:16:12.576937Z","shell.execute_reply":"2023-01-14T07:16:12.590416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isnull()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:16:23.486093Z","iopub.execute_input":"2023-01-14T07:16:23.486599Z","iopub.status.idle":"2023-01-14T07:16:23.608845Z","shell.execute_reply.started":"2023-01-14T07:16:23.486552Z","shell.execute_reply":"2023-01-14T07:16:23.607798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:16:41.896634Z","iopub.execute_input":"2023-01-14T07:16:41.897032Z","iopub.status.idle":"2023-01-14T07:16:41.955971Z","shell.execute_reply.started":"2023-01-14T07:16:41.897002Z","shell.execute_reply":"2023-01-14T07:16:41.954715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pixel_array(dataset, figsize=(5,5)):\n    plt.figure(figsize=figsize)\n    plt.grid(False)\n    plt.imshow(dataset.pixel_array, cmap='gray') # cmap=plt.cm.bone)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:32.157440Z","iopub.execute_input":"2023-01-14T06:49:32.157840Z","iopub.status.idle":"2023-01-14T06:49:32.164592Z","shell.execute_reply.started":"2023-01-14T06:49:32.157810Z","shell.execute_reply":"2023-01-14T06:49:32.163311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imdir = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00123637202217151272140\"\nprint(\"total images for patient ID00123637202217151272140: \", len(os.listdir(imdir)))\nfig=plt.figure(figsize=(12, 12))\ncolumns = 4\nrows = 5\nimglist = os.listdir(imdir)\nfor i in range(1, columns*rows +1):\n    filename = imdir + \"/\" + str(i) + \".dcm\"\n    ds = pydicom.dcmread(filename)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(ds.pixel_array, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:38.191898Z","iopub.execute_input":"2023-01-14T06:49:38.192747Z","iopub.status.idle":"2023-01-14T06:49:40.712878Z","shell.execute_reply.started":"2023-01-14T06:49:38.192702Z","shell.execute_reply":"2023-01-14T06:49:40.711517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imdir = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/ID00123637202217151272140\"\nprint(\"total images for patient ID00123637202217151272140: \", len(os.listdir(imdir)))\nfig=plt.figure(figsize=(12, 12))\ncolumns = 4\nrows = 5\nimglist = os.listdir(imdir)\nfor i in range(1, columns*rows +1):\n    filename = imdir + \"/\" + str(i) + \".dcm\"\n    ds = pydicom.dcmread(filename)\n    fig.add_subplot(rows, columns, i)\n    plt.imshow(ds.pixel_array, cmap='jet')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:44.040915Z","iopub.execute_input":"2023-01-14T06:49:44.041378Z","iopub.status.idle":"2023-01-14T06:49:46.918168Z","shell.execute_reply.started":"2023-01-14T06:49:44.041338Z","shell.execute_reply":"2023-01-14T06:49:46.916847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_data_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:50.811364Z","iopub.execute_input":"2023-01-14T06:49:50.811781Z","iopub.status.idle":"2023-01-14T06:49:50.873158Z","shell.execute_reply.started":"2023-01-14T06:49:50.811748Z","shell.execute_reply":"2023-01-14T06:49:50.872055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostRegressor, CatBoostClassifier","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:54.068658Z","iopub.execute_input":"2023-01-14T06:49:54.069736Z","iopub.status.idle":"2023-01-14T06:49:55.425898Z","shell.execute_reply.started":"2023-01-14T06:49:54.069688Z","shell.execute_reply":"2023-01-14T06:49:55.424571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = ['Patient', 'Sex', 'SmokingStatus']\n\ny_train, X_train = train_df['FVC'], train_df.drop(['FVC'], axis=1)\ny_test, X_test = test_df['FVC'], test_df.drop(['FVC'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:49:58.088481Z","iopub.execute_input":"2023-01-14T06:49:58.089750Z","iopub.status.idle":"2023-01-14T06:49:58.099584Z","shell.execute_reply.started":"2023-01-14T06:49:58.089701Z","shell.execute_reply":"2023-01-14T06:49:58.098381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CatBoostClassifier(iterations=5,\n                           random_seed=0,\n                           learning_rate=0.1,\n                           custom_loss=['Accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:50:05.988514Z","iopub.execute_input":"2023-01-14T06:50:05.988937Z","iopub.status.idle":"2023-01-14T06:50:05.998163Z","shell.execute_reply.started":"2023-01-14T06:50:05.988887Z","shell.execute_reply":"2023-01-14T06:50:05.996550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, y_train,\n          eval_set=(X_test, y_test),\n          cat_features=cat_features,\n          plot=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T06:50:10.098458Z","iopub.execute_input":"2023-01-14T06:50:10.098874Z","iopub.status.idle":"2023-01-14T07:02:55.612129Z","shell.execute_reply.started":"2023-01-14T06:50:10.098841Z","shell.execute_reply":"2023-01-14T07:02:55.611038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fea_imp = pd.DataFrame({'importance': model.feature_importances_, 'col': model.feature_names_})\nfea_imp = fea_imp.sort_values(['importance', 'col'], ascending=[True, False]).iloc[-40:]\nfea_imp.plot(kind='barh', x='col', y='importance', figsize=(10, 10))","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:32:54.883706Z","iopub.execute_input":"2023-01-14T07:32:54.884198Z","iopub.status.idle":"2023-01-14T07:32:55.155638Z","shell.execute_reply.started":"2023-01-14T07:32:54.884164Z","shell.execute_reply":"2023-01-14T07:32:55.154385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_sample_submission, X_sample_submission = train_df['FVC'], train_df.drop(['FVC'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:32:59.608048Z","iopub.execute_input":"2023-01-14T07:32:59.608466Z","iopub.status.idle":"2023-01-14T07:32:59.615793Z","shell.execute_reply.started":"2023-01-14T07:32:59.608432Z","shell.execute_reply":"2023-01-14T07:32:59.614615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CatBoostClassifier(iterations=5,\n                           random_seed=0,\n                           learning_rate=0.1,\n                           custom_loss=['Accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:33:01.532638Z","iopub.execute_input":"2023-01-14T07:33:01.533082Z","iopub.status.idle":"2023-01-14T07:33:01.539018Z","shell.execute_reply.started":"2023-01-14T07:33:01.533046Z","shell.execute_reply":"2023-01-14T07:33:01.537626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_sample_submission, y_sample_submission,\n          eval_set=(X_sample_submission, y_sample_submission),\n          cat_features=cat_features,\n          plot=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:33:04.221103Z","iopub.execute_input":"2023-01-14T07:33:04.221580Z","iopub.status.idle":"2023-01-14T07:45:20.728587Z","shell.execute_reply.started":"2023-01-14T07:33:04.221540Z","shell.execute_reply":"2023-01-14T07:45:20.727353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fea_imp = pd.DataFrame({'importance': model.feature_importances_, 'col': model.feature_names_})\nfea_imp = fea_imp.sort_values(['importance', 'col'], ascending=[True, False]).iloc[-40:]\nfea_imp.plot(kind='barh', x='col', y='importance', figsize=(10, 10))","metadata":{"execution":{"iopub.status.busy":"2023-01-14T07:50:55.103209Z","iopub.execute_input":"2023-01-14T07:50:55.103906Z","iopub.status.idle":"2023-01-14T07:50:55.375264Z","shell.execute_reply.started":"2023-01-14T07:50:55.103852Z","shell.execute_reply":"2023-01-14T07:50:55.373588Z"},"trusted":true},"execution_count":null,"outputs":[]}]}