{"cells":[{"metadata":{},"cell_type":"markdown","source":"# How Well Can We Do Without A Model?\n\n## Imports","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set()\nimport gc; gc.enable()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Ingestion","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ndf_test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\n\nprint('Train shape: ', df_train.shape)\nprint('Number of unique customers in train: {}'.format(df_train['Patient'].nunique()))\nprint('Test shape:', df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Quick Peek","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Relative-Group FVC Distributions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = (20,8)\n\nplt.figure(figsize=SIZE)\n(df_train.FVC / df_train.Percent * 100).hist()\nplt.show()\n\nplt.figure(figsize=SIZE)\n(df_test.FVC / df_test.Percent * 100).hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Prep","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Patient'] = sub.Patient_Week.str.split('_').apply(lambda x: x[0])\nsub['Weeks'] = sub.Patient_Week.str.split('_').apply(lambda x: int(x[1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Sanity Check","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.Patient.unique().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.Patient.unique().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Weeks'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.Weeks.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.Weeks.hist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Outliers? Poorly-Behaved Patient Percents\n\nI am choosing to focus on Percent over FVC since FVC is dependent on hidden parameters: weight, height, ethnicity, etc.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patients = df_train.Patient.unique().tolist()\n\nplt.figure(figsize=SIZE)\nfor patient in patients:\n    temp = df_train[df_train.Patient == patient][['Percent', 'Weeks']].copy()\n    plt.plot(temp.Weeks, temp.Percent.pct_change())\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=SIZE)\n(df_train.Percent).hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=SIZE)\nfor patient in patients:\n    temp = df_train[df_train.Patient == patient][['Percent', 'Weeks']].copy()\n    plt.plot(temp.Weeks, temp.Percent)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patients = df_test.Patient.unique().tolist()\n\nplt.figure(figsize=SIZE)\nfor patient in patients:\n    temp = df_test[df_test.Patient == patient][['Percent', 'Weeks']].copy()\n    plt.scatter(temp.Weeks, temp.Percent)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Combine DataFrames","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['FVC'] = np.nan\nsub['Confidence'] = np.nan\n\ndf = pd.concat([df_train, df_test, sub], axis=0, ignore_index=True)\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Simple / Naive Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df['gpFVC'] = df.FVC / df.Percent * 100\n\ndf['Percent'] = df['Percent'].fillna(df.groupby('Patient')['Percent'].transform('median'))\ndf['gpFVC'] = df['gpFVC'].fillna(df.groupby('Patient')['gpFVC'].transform('median'))\n\ndf['FVC'] = df.Percent * df.gpFVC / 100\n\ndf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"del sub['FVC']; del sub['Confidence']; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = sub.merge(df[['Patient_Week', 'FVC']], on='Patient_Week', how='left')\ndel sub['Patient']\ndel sub['Weeks']\nsub['Confidence'] = sub['FVC']*0.12\n\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}