{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Input,Dense,Dropout,Lambda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lets work with csv files\n\ntrain_df=pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntest_df=pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\nsub_df=pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df['Patient'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby(['SmokingStatus']).count()['Sex']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_train_df=set(train_df['Patient'].unique())\npatient_test_df= set(test_df['Patient'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_train_df.intersection(patient_test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_train_df=train_df[['Patient', 'Age', 'Sex', 'SmokingStatus']].drop_duplicates()\nunique_train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Patient'].value_counts().max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Patient'].value_counts().min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[train_df['Patient']=='ID00422637202311677017371']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n#1 visualization\n\n#smoking status\n\nunique_train_df['SmokingStatus'].value_counts().plot(kind='bar', \n                                              color='blue',\n                                              title='Distribution of the SmokingStatus column')\n\nplt.ylabel('counts')\nplt.xticks(rotation = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Weeks'].value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Weeks'].value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#week distribution\nplt.figure(figsize=(50,40))\ntrain_df['Weeks'][:int(len(train_df['Weeks'].unique())/2)].value_counts().plot(kind='barh', color='blue',\n                                              title='Distribution of the weeks column')\nplt.tick_params(axis=\"x\", labelsize=20)\nplt.tick_params(axis=\"y\", labelsize=20)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['FVC'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fvc vs percent \nimport plotly.express as px\nplt.figure(figsize=(30,20))\npx.scatter(train_df,x='Percent',y='FVC',color='Age')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#FVC vs age\n\npx.bar(train_df,x='SmokingStatus',y='FVC',color='Age')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#percent\n\ntrain_df['Percent'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(train_df['Percent'],train_df['FVC'],color='red')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train_df.append([test_df, sub_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df),len(test_df),len(sub_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.concat( (train_df,test_df) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\nlabel_encoder=preprocessing.LabelEncoder()\ntrain_df['SmokingStatus']=label_encoder.fit_transform(train_df['SmokingStatus'])\ntrain_df['Sex']=label_encoder.fit_transform(train_df['Sex'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Percent']       = (train_df['Percent'] - train_df['Percent'].mean()) / train_df['Percent'].std()\ntrain_df['Age']           = (train_df['Age'] - train_df['Age'].mean()) / train_df['Age'].std()\ntrain_df['Sex']           = (train_df['Sex'] - train_df['Sex'].mean()) / train_df['Sex'].std()\ntrain_df['SmokingStatus'] = (train_df['SmokingStatus'] - train_df['SmokingStatus'].mean()) / train_df['SmokingStatus'].std()\ntrain_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#keras model\n\n# model architecture https://www.kaggle.com/chrisden/6-82-quantile-reg-lr-schedulers-checkpoints\n\nfrom tensorflow.keras.models import Model\n\ni = Input(shape=(5,))\nx = Dense(100, activation=\"relu\", name=\"d1\")(i)\nx = Dense(100, activation=\"relu\", name=\"d2\")(x)\np1 = Dense(3, activation=\"linear\", name=\"p1\")(x)\np2 = Dense(3, activation=\"relu\", name=\"p2\")(x)\npreds=preds =Lambda(lambda x: x[0] + tf.cumsum(x[1], axis = 1), \n                     name = \"preds\")([p1, p2])\n\n\nmodel=Model(i,[p1,p2])\nmodel.compile(optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False),loss='mse')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"r=model.fit(train_df[['Weeks','Percent','Sex','SmokingStatus','Age']],train_df['FVC'],epochs=100,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#better option","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#quantile regression model\n#linear model\n\nfrom statsmodels.formula.api import quantreg\n\nmodel_1=quantreg('FVC ~ Weeks+Sex+Age+SmokingStatus+Percent', train_df).fit(q=0.15)\nmodel_2=quantreg('FVC ~ Weeks+Sex+Age+SmokingStatus+Percent', train_df).fit(q=0.50)\nmodel_3=quantreg('FVC ~ Weeks+Percent+Age+Sex+SmokingStatus', train_df).fit( q=0.85 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['FVC1'],train_df['FVC2'],train_df['FVC3']=model_1.predict(train_df),model_2.predict(train_df),model_3.predict(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(sub_df),len(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['Confidence1'] = train_df.iloc[:730,9] - train_df.iloc[:730,8]\ndf['Patient_Week']=train_df['Patient'][:730]+'_'+train_df['Weeks'].astype('str')[:730]\ndf['Confidence']=sub_df['Confidence']\ndf['FVC']=train_df['FVC'][:730]\ndf['FVC1']=train_df['FVC1'][:730]\n# get rid of unused data and show some non-empty data\nsubmission = df[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()\nsubmission.loc[~submission.FVC1.isnull()].head(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}