{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ndf_test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Check dimension","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train_dim',df_train.shape)\nprint('test_dim',df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_test.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. check number of patients","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Total patients:',len(df_train.Patient.unique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# not all week are depicted and negative week is exist","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.groupby(['Patient','Weeks'])['Weeks'].count().head(20))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"at week1 only 5 patient data in train but image data able to find week 1","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.groupby(['Weeks'])['Patient'].count().head(20))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"check target variable: FGV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(df_train['FVC'],bins=30)\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['Patient'].unique()[1:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfor i in df_train['Patient'].unique()[0:5]:\n    plt.figure(figsize=(20, 4))\n    plt.plot(df_train[df_train['Patient']==i]['Weeks'],df_train[df_train['Patient']==i]['FVC'])\n    plt.ylabel('FVC')\n    plt.xlabel('weeks')\n    a=df_train[df_train['Patient']==i]['Weeks'].max() - df_train[df_train['Patient']==i]['Weeks'].min()\n    plt.title(i +' FVC in '+str(a)+' Weeks')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- roughly have 9 measurement per patient. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.groupby(['Patient'])['Weeks'].count().describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- Other features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20, 14))\nplt.subplot(2,2,1)\nplt.title('Mean Age versus FVC')\nplt.plot(df_train.groupby(['Age'])['FVC'].mean())\nplt.subplot(2,2,2)\nplt.title('Min Age versus FVC')\nplt.plot(df_train.groupby(['Age'])['FVC'].min())\nplt.subplot(2,2,3)\nplt.title('Max Age versus FVC')\nplt.plot(df_train.groupby(['Age'])['FVC'].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.Sex.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(df_train.Age)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(df_train.FVC,df_train.Percent,  color='gray')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub=pd.DataFrame()\nsub['patient']=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')['Patient_Week'].str.split('_', n = 1, expand = True)[0]\nsub['week']=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')['Patient_Week'].str.split('_', n = 1, expand = True)[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['patient'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['week'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features= df_train.drop(['Patient','FVC','Percent'],axis=1)\nX=pd.get_dummies(features)\ny=df_train.FVC\n#y=df_train.Percent","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split \nfrom sklearn.linear_model import LinearRegression\nfrom sklearn import metrics\n%matplotlib inline\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=11)\nregressor = LinearRegression()  \nregressor.fit(X_train, y_train) #training the algorithm\ny_pred = regressor.predict(X_test)\ndf = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})\nplt.scatter(y_pred, y_test,  color='gray')\nplt.show()\nprint('Mean Absolute Error:',metrics.mean_absolute_error(y_test, y_pred))  \nprint('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  \nprint('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.linear_model import Ridge\n# from sklearn.ensemble import AdaBoostRegressor\n# from sklearn.ensemble import GradientBoostingRegressor\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=11)\n# ls=[]\n# lr=[]\n# mae=[]\n# mse=[]\n# rmse=[]\n# estimator=[]\n# mss=[]\n# ss=[]\n# md=[]\n# for loss in ['ls', 'lad', 'huber', 'quantile']:\n#     for learning_rate in [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]:\n#         for n_estimators in [10,20,30,50,100,120]:\n#             for subsample in [0.1,0.2,0.3,0.5,0.7,0.9]:\n#                 for min_samples_split in [2,3,5,7,10,12]:\n#                     for max_depth in [3,5,7,10,12]:\n#                         ls.append(loss)\n#                         lr.append(learning_rate)\n#                         estimator.append(n_estimators)\n#                         ss.append(subsample)\n#                         mss.append(min_samples_split)\n#                         md.append(max_depth)\n#                         clf = GradientBoostingRegressor(random_state=0,loss=loss,learning_rate=learning_rate,n_estimators=n_estimators,\n#                                                        subsample=subsample,min_samples_split=min_samples_split,max_depth=max_depth)\n#                         clf.fit(X_train, y_train) #training the algorithm\n#                         y_pred = clf.predict(X_test)\n#                         #df = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})\n#                         #plt.scatter(y_pred, y_test,  color='gray')\n#                         #plt.show()\n#                         mae.append(metrics.mean_absolute_error(y_test, y_pred))\n#                         mse.append(metrics.mean_squared_error(y_test, y_pred))\n#                         rmse.append( np.sqrt(metrics.mean_squared_error(y_test, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=11)\nclf = GradientBoostingRegressor(random_state=0)\nclf.fit(X_train, y_train) #training the algorithm\ny_pred = clf.predict(X_test)\nplt.scatter(y_pred, y_test,  color='gray')\nplt.show()\nprint('Mean Absolute Error:',metrics.mean_absolute_error(y_test, y_pred))  \nprint('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  \nprint('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_mod=pd.DataFrame()\npat=[]\nfor i in df_test.Patient:\n    for j in sub['week'].unique(): \n        pat.append(i)\n        df_test.loc[df_test.Patient==i, 'Weeks'] = j\n        test_mod=test_mod.append(df_test[df_test.Patient == i])\ntest_mod['Weeks']=test_mod['Weeks'].astype(str).astype(int)\ntest=pd.get_dummies(test_mod.drop(['Patient','FVC','Percent'],axis=1))\ntest2=test.drop(['SmokingStatus_Ex-smoker','SmokingStatus_Never smoked','Sex_Male'],axis=1)\ntest2['Sex_Female']=0\ntest2['Sex_Male']=test['Sex_Male']\ntest2['SmokingStatus_Currently smokes']=0\ntest2['SmokingStatus_Ex-smoker']=test['SmokingStatus_Ex-smoker']\ntest2['SmokingStatus_Never smoked']=test['SmokingStatus_Never smoked']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test2['pred']=regressor.predict(test2)\ntest2['Patient']=pat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test2['Patient_Week']=test2[\"Patient\"].str.cat(test2['Weeks'].astype(str),sep='_') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.DataFrame()\nsubmission['Patient_Week']=test2['Patient_Week']\nsubmission['FVC']=test2['pred']\nsubmission['Confidence']=100\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# result=pd.DataFrame()\n# result['loss']=ls\n# result['learning_rate']=lr\n# result['estimator']=estimator\n# result['subsample']=ss\n# result['min_samples_split']=mss\n# result['max_depth']=md\n# result['mae']=mae\n# result['mse']=mse\n# result['rmse']=rmse","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#result[result.mae==result.mae.min()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.ensemble import RandomForestRegressor\n# clf =  RandomForestRegressor(max_depth=2, random_state=0)\n# clf.fit(X_train, y_train) #training the algorithm\n# y_pred = clf.predict(X_test)\n# df = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})\n# plt.scatter(y_pred, y_test,  color='gray')\n# plt.show()\n# print('Mean Absolute Error:',metrics.mean_absolute_error(y_test, y_pred))  \n# print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  \n# print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test=pd.get_dummies(df_test.drop(['Patient','FVC','Percent'],axis=1))\n# test2=test.drop(['SmokingStatus_Ex-smoker','SmokingStatus_Never smoked','Sex_Male'],axis=1)\n# test2['Sex_Female']=0\n# test2['Sex_Male']=test['Sex_Male']\n# test2['SmokingStatus_Currently smokes']=0\n# test2['SmokingStatus_Ex-smoker']=test['SmokingStatus_Ex-smoker']\n# test2['SmokingStatus_Never smoked']=test['SmokingStatus_Never smoked']","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}