{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom statsmodels.formula.api import quantreg\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv( '../input/osic-pulmonary-fibrosis-progression/train.csv' )\ntest  = pd.read_csv( '../input/osic-pulmonary-fibrosis-progression/test.csv' )\n\ntrain['traintest'] = 0\ntest ['traintest'] = 1\n\nsub   = pd.read_csv( '../input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )\nsub['Weeks']   = sub['Patient_Week'].apply( lambda x: int(x.split('_')[-1]) )\nsub['Patient'] = sub['Patient_Week'].apply( lambda x: x.split('_')[0] ) \n\nprint( train.shape, test.shape, sub.shape )\n\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.tail(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.tail(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train have 176 patients and Test only 5\n# Welcome to the uncertainty and LB shake up world!","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.Patient.nunique(), sub.Patient.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.Patient.isin( test.Patient.unique() ).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Concatenate the 5 samples we have in test with the train set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat( (train,test) )\ntrain.sort_values( ['Patient','Weeks'], inplace=True )\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Age'].hist( bins=100, figsize=(5, 5) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Percent'].hist( bins=100, figsize=(5, 5) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['FVC'].hist( bins=100, figsize=(5, 5) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby( ['Sex','SmokingStatus'] )['FVC'].agg( ['mean','std','count'] )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby( ['Sex','Age'] )['FVC'].agg( ['mean','std','count'] )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Label encode Strings","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Sex']           = pd.factorize( train['Sex'] )[0]\ntrain['SmokingStatus'] = pd.factorize( train['SmokingStatus'] )[0]\ntrain","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Manual mean removal + std","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Percent']       = (train['Percent'] - train['Percent'].mean()) / train['Percent'].std()\ntrain['Age']           = (train['Age'] - train['Age'].mean()) / train['Age'].std()\ntrain['Sex']           = (train['Sex'] - train['Sex'].mean()) / train['Sex'].std()\ntrain['SmokingStatus'] = (train['SmokingStatus'] - train['SmokingStatus'].mean()) / train['SmokingStatus'].std()\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fit quantreg models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"modelL = quantreg('FVC ~ Weeks+Percent+Age+Sex+SmokingStatus', train).fit( q=0.15 )\nmodel  = quantreg('FVC ~ Weeks+Percent+Age+Sex+SmokingStatus', train).fit( q=0.50 )\nmodelH = quantreg('FVC ~ Weeks+Percent+Age+Sex+SmokingStatus', train).fit( q=0.85 )\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ypredL'] = modelL.predict( train ).values\ntrain['ypred']  = model.predict( train ).values\ntrain['ypredH'] = modelH.predict( train ).values\ntrain['ypredstd'] = 0.5*np.abs(train['ypredH'] - train['ypred'])+0.5*np.abs(train['ypred'] - train['ypredL'])\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Calculate competition metric","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric( trueFVC, predFVC, predSTD ):\n    \n    clipSTD = np.clip( predSTD, 70 , 9e9 )  \n    \n    deltaFVC = np.clip( np.abs(trueFVC-predFVC), 0 , 1000 )  \n\n    return np.mean( -1*(np.sqrt(2)*deltaFVC/clipSTD) - np.log( np.sqrt(2)*clipSTD ) )\n    \n\nprint( 'Metric:', metric( train['FVC'].values, train['ypred'].values, train['ypredstd'].values  ) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge features to test set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dt = train.loc[ train.traintest==1 ,['Patient','Percent','Age','Sex','SmokingStatus']]\ntest = pd.merge( sub, dt, on='Patient', how='left' )\ntest.sort_values( ['Patient','Weeks'], inplace=True )\ntest.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['ypredL'] = modelL.predict( test ).values\ntest['FVC']    = model.predict( test ).values\ntest['ypredH'] = modelH.predict( test ).values\ntest['Confidence'] = np.abs(test['ypredH'] - test['ypredL']) / 2\n\ntest.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[['Patient_Week','FVC','Confidence']].to_csv('submission.csv', index=False)\ntest[['Patient_Week','FVC','Confidence']].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.loc[ test.Patient=='ID00419637202311204720264'  ].plot( x='Weeks', y='FVC' )\ntest.loc[ test.Patient=='ID00421637202311550012437'  ].plot( x='Weeks', y='FVC' )\ntest.loc[ test.Patient=='ID00422637202311677017371'  ].plot( x='Weeks', y='FVC' )\ntest.loc[ test.Patient=='ID00423637202312137826377'  ].plot( x='Weeks', y='FVC' )\ntest.loc[ test.Patient=='ID00426637202313170790466'  ].plot( x='Weeks', y='FVC' )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.loc[ test.Patient=='ID00419637202311204720264'  ].plot( x='Weeks', y='Confidence' )\ntest.loc[ test.Patient=='ID00421637202311550012437'  ].plot( x='Weeks', y='Confidence' )\ntest.loc[ test.Patient=='ID00422637202311677017371'  ].plot( x='Weeks', y='Confidence' )\ntest.loc[ test.Patient=='ID00423637202312137826377'  ].plot( x='Weeks', y='Confidence' )\ntest.loc[ test.Patient=='ID00426637202313170790466'  ].plot( x='Weeks', y='Confidence' )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}