{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom pandas_profiling import ProfileReport\nimport seaborn as sns\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading data\ntest_df = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsample_sub= pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\ntrain_df=pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pandas_profiling import ProfileReport \nProfileReport(train_df,progress_bar=False) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# How to read/plot an image\n\n#pydicom: Python library for Dicom (Digital Imaging in Medicine)\nimport pydicom\nimport matplotlib.pyplot as plt\n\nfig=plt.figure(figsize=(3, 3))\nfilename = '../input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/10.dcm'\nds = pydicom.dcmread(filename)\n# fig.add_subplot(rows, columns, i)\nplt.imshow(ds.pixel_array, cmap='gray')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Patients in test set are also in the training set\n\nprint(test_df.Patient.nunique())\nprint ( train_df.Patient.nunique())\npatients_train_ids= train_df.Patient.unique()\npatient_test_list= test_df.Patient.unique()\nlist_p = train_df.Patient.isin( patient_test_list )\nprint(train_df[list_p].Patient.unique())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Statistics**\n"},{"metadata":{},"cell_type":"markdown","source":"### Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get total number of images + nb of images per patient\n\nnb_image_patient={}\ntot=0\n\nfor ids in patients_train_ids:\n    path =f'../input/osic-pulmonary-fibrosis-progression/train/{ids}'\n    nb_image_patient[ids] = len(list(os.listdir(path)))\n    liste = list(os.listdir(path))\n    tot+= len(liste)\n    nb_image_patient[ids] = len(liste)\nprint (\"Total number of images in training set: \\n\", tot)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(nb_image_patient.values(),bins=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Categorical variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_sex=round(pd.crosstab(train_df.Sex,columns=\"count\")/len(train_df.Sex),2)\nlabels = list(freq_sex.index)\nsizes = list(freq_sex['count'])\n\nfig, ax= plt.subplots(1,2,figsize=(15,5))\nax[0].pie(sizes, labels=labels, autopct='%1.1f%%',\n        shadow=False, startangle=90, colors=['lightgrey','lightblue'])\nax[0].axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nax[0].set_title('Sex')\n\n\nfreq_smoke=round(pd.crosstab(train_df.SmokingStatus,columns=\"count\")/len(train_df.SmokingStatus),2)\nlabels = list(freq_smoke.index)\nsizes = list(freq_smoke['count'])\n\nax[1].pie(sizes, labels=labels, autopct='%1.1f%%',\n        shadow=False, startangle=90, colors=['lightgrey','lightblue','mediumblue'])\nax[1].axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nax[1].set_title('Smoking Status')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x='Sex', hue='SmokingStatus', palette=\"ch:.25\", data=train_df);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Numerical variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(121)\npd.DataFrame.boxplot(train_df[['Age']])\nplt.subplot(122)\npd.DataFrame.boxplot(train_df[['FVC']])\nplt.subplots_adjust( hspace=0.25,wspace=0.55)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Details of variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Age\nprint('Minimum aged patient:',min(train_df['Age']))\nprint('Maximum aged patient:',max(train_df['Age']),'\\n')\n\nfig = plt.figure()\nsns.distplot(train_df['Age'])\nplt.title('Age Distribution')\nplt.xlabel('Age',size=12)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FVC - the recorded lung capacity in ml = volume d'air expiré en une haleine\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Minimum FVC value:',min(train_df['FVC']))\nprint('Maximum FVC value:',max(train_df['FVC']),'\\n')\n\nfig, axes = plt.subplots(1, 3, figsize = (20, 6))\nsns.distplot(train_df['FVC'], ax = axes[0])\naxes[0].set_title('FVC Distribution')\nplt.xlabel('FVC',size=12)\n\n#Percent\nprint('Minimum Percentage:',min(train_df['Percent']))\nprint('Maximum Percentage:',max(train_df['Percent']))\n\nsns.distplot(train_df['Percent'], ax = axes[1])\naxes[1].set_title('Percentage Distribution')\nplt.xlabel('Percent',size=12)\n\nsns.scatterplot(data = train_df, x=\"FVC\", y=\"Percent\", hue = 'Sex',ax = axes[2])\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## FVC evolution for 10 patients"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Weeks_norm'] = train_df.groupby(by = 'Patient').Weeks.agg('diff')\ntrain_df['Weeks_norm'].replace(np.nan,0,inplace=True)\ntrain_df['Weeks_norm'] = train_df.groupby(by = 'Patient').Weeks_norm.agg('cumsum')\n\ntrain_df['Visit'] = 1\ntrain_df['Visit'] = train_df.groupby(by = 'Patient').Visit.cumsum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient[\"Sex\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize = (20, 6))\n\ndeg= 1\npat = train_df.Patient.sample(n=10)\n\npatient = train_df[train_df.Patient.isin(pat)]\n\nsns.lineplot(data = patient, x=\"Weeks_norm\", y=\"FVC\", hue='Patient', ax = axes[0])\naxes[0].set_title('FVC Evolution for the patients of the sample')\n\nsns.lineplot(data = train_df, x=\"Weeks_norm\", y=\"FVC\", hue = 'Sex', ax = axes[1])\naxes[1].set_title('FVC Evolution per Sex')\n\ndff = patient[patient['Sex'] == 'Female'].sort_values(by = 'Weeks_norm')\ndfm = patient[patient['Sex'] == 'Male'].sort_values(by = 'Weeks_norm')\n\nXf = dff['Weeks_norm']\nyf= dff['FVC']\n\nXm= dfm['Weeks_norm']\nym= dfm['FVC']\n\ncf = np.polyfit(Xf, yf,deg )\ncm = np.polyfit( Xm, ym,deg )\n\np = np.poly1d(cm)\nplt.plot(Xm, p(Xm))\np = np.poly1d(cf)\nplt.plot(Xf, p(Xf))\nplt.xlabel('Weeks')\nplt.ylabel('FVC')\nplt.title('FVC evolution per sex, smoothed')\nplt.grid()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visites = train_df.groupby(by = 'Patient')['Weeks'].count()\nsns.barplot(visites.index,visites)\nplt.title('Nombre de visites par patient')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoding categorical variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Replacing categorical variable 'Sex' for training set\na = train_df[train_df.loc[:,'Sex']== 'Female'].index.tolist()\ntrain_df.loc[a,'Sex']=1\na = train_df[train_df.loc[:,'Sex']== 'Male'].index.tolist()\ntrain_df.loc[a,'Sex']=0\n\n# Replacing categorical variable 'Sex' for test set\n##\na = test_df[test_df.loc[:,'Sex']== 'Male'].index.tolist()\ntest_df.loc[a,'Sex']= 0\n#\na = test_df[test_df.loc[:,'Sex']== 'Female'].index.tolist()\ntest_df.loc[a,'Sex']= 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Replacing categorical variable 'SmokingStatus' for both set\nSmokingCategories = train_df.SmokingStatus.unique()\nprint(SmokingCategories)\n# 3 different categories and no none type\ni = 0\nfor cat in SmokingCategories:\n    a = train_df[train_df.loc[:,'SmokingStatus']== cat].index.tolist()\n    b = test_df[test_df.loc[:,'SmokingStatus']== cat].index.tolist()\n    train_df.loc[a,'SmokingStatus']= i\n    test_df.loc[b,'SmokingStatus']= i\n    i+=1","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}