{"cells":[{"metadata":{},"cell_type":"markdown","source":"# OSIC: Pulmonary Fibrosis | Baseline Regression\n\n* Feature engineering\n* Experimentation with simple and multiple regression models\n* Preparation of sample submissions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Import Libraries","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Visualization\nimport matplotlib.pyplot as plt \nimport seaborn as sns\n\nfrom sklearn import linear_model\nimport statsmodels.api as sm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering | Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Location of the training images\nBASE_PATH = '../input/osic-pulmonary-fibrosis-progression'\n\n# image directories\ndata_train_dir = f'{BASE_PATH}/train'\ndata_test_dir = f'{BASE_PATH}/test'\n\n# Location of training labels\ntrain = pd.read_csv(f'{BASE_PATH}/train.csv')\ntest = pd.read_csv(f'{BASE_PATH}/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create 'Initial FVC' Column","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_dict = {}\ndef init_fvc(row):\n    if row['Patient'] not in patient_dict.keys():\n        patient_dict[row['Patient']] = row['FVC']\n        return row['FVC']\n    else:\n        return patient_dict[row['Patient']]\n\ntrain['InitFVC'] = train.apply(lambda row: init_fvc(row), axis=1)\ntrain.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create 'Initial Week' Column","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_dict = {}\ndef init_week(row):\n    if row['Patient'] not in patient_dict.keys():\n        patient_dict[row['Patient']] = row['Weeks']\n        return row['Weeks']\n    else:\n        return patient_dict[row['Patient']]\n\ntrain['InitWeeks'] = train.apply(lambda row: init_week(row), axis=1)\ntrain.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Add Column 'InitPercent'","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patient_dict = {}\ndef init_percent(row):\n    if row['Patient'] not in patient_dict.keys():\n        patient_dict[row['Patient']] = row['Percent']\n        return row['Percent']\n    else:\n        return patient_dict[row['Patient']]\n\ntrain['InitPercent'] = train.apply(lambda row: init_percent(row), axis=1)\ntrain.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Convert Sex and SmokingStatus to Indicator Variables","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.get_dummies(train, columns=['Sex', 'SmokingStatus'], prefix=['Sex', 'SmokingStatus'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering | Test\n* For each patient, add a row for weeks with range -12 to 133. \n* All else is *constant* for now\n* Age is *initial age*","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = []\nfor i in range(-12, 133+1):\n    for index, row in test.iterrows():\n        new_cols = list(test.columns)\n        new_cols.append('InitWeeks')\n        new_vals = [row['Patient'], i, row['FVC'], row['Percent'],row['Age'],row['Sex'],row['SmokingStatus'], row['Weeks']]\n        data.append(dict(zip(new_cols, new_vals)))\ntest_df = pd.DataFrame(data)\ntest_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Regression Models","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## 1. Naive Simple Linear Regression (FVC vs. Weeks)","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Naive Simple Linear Regression (FVC vs. Weeks)\nX = train[['Weeks']]\nY = train['FVC']\n\nregr = linear_model.LinearRegression()\nregr.fit(X, Y)\n\nprint('Intercept: \\n', regr.intercept_)\nprint('Coefficients: \\n', regr.coef_)\n\nmodel = sm.OLS(Y, X).fit()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create Weeks vs. FVC Plot \n\nTrend is very weak because of the different initial FVC for each patient. (Everyone has different starting points, and decline at different rates). \n\n* A negative correlation can be visualized by **grouping by InitFVC** (See section 2 below). ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.lmplot(x='Weeks', y='FVC', data=train.sample(frac=0.8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"X_test = test[['Weeks']]\nY_test = test['FVC']\nprint('Predicted FVCs: \\n', regr.predict(X_test))\nprint('Actual FVCs: \\n', Y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Basic Multiple Linear Regression (FVC vs. InitFVC & Weeks)\n* Incorporate InitFVC as a feature in the linear regression model in order to differentiate beteween each patient's starting points. \n* InitFVC controls y-intercept of line. ","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Naive Simple Linear Regression (FVC vs. Weeks)\nX = train[['Weeks', 'InitFVC']]\nY = train['FVC']\n\nregr = linear_model.LinearRegression()\nregr.fit(X, Y)\n\nprint('Intercept: \\n', regr.intercept_)\nprint('Coefficients: \\n', regr.coef_)\n\nmodel = sm.OLS(Y, X).fit()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize Weeks vs. FVC Grouped by InitFVC\n* When grouped by InitFVC, the negative correlation is much more pronounced. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.lmplot(x='Weeks', y='FVC', hue='InitFVC', data=train.head(98))\n\ndisplay(test.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"X_test = test_df[['Weeks', 'FVC']]\nY_test = test_df['FVC']\n\nY_pred = regr.predict(X_test)\n# print('Predicted FVCs: \\n', Y_pred)\n# print('Actual FVCs: \\n', Y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Basic Multiple Linear Regression (FVC vs. InitFVC, Weeks, InitWeeks)","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Multiple Linear Regression \nX = train[['Weeks', 'InitFVC', 'InitWeeks']]\nY = train['FVC']\n\nregr = linear_model.LinearRegression()\nregr.fit(X, Y)\n\nprint('Intercept: \\n', regr.intercept_)\nprint('Coefficients: \\n', regr.coef_)\n\nmodel = sm.OLS(Y, X).fit()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"X_test = test_df[['Weeks', 'FVC', 'InitWeeks']]\nY_test = test_df['FVC']\n\nY_pred = regr.predict(X_test)\n# print('Predicted FVCs: \\n', Y_pred)\n# print('Actual FVCs: \\n', Y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize Test Regression \n* Slopes are equal for this model, since **InitFVC** and **InitWeeks** are *constant* for each patient","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# use df.iloc[idx] to access specific row \ndata = []\nfor i in range(test_df.shape[0]):\n    new_cols = ['Patient', 'Weeks', 'FVC', 'Confidence']\n    new_vals = [test_df.iloc[i]['Patient'], test_df.iloc[i]['Weeks'], Y_pred[i], 100]\n    data.append(dict(zip(new_cols, new_vals)))\nviz = pd.DataFrame(data)\nsns.lmplot(x='Weeks', y='FVC', hue='Patient', data=viz)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. MLR (FVC vs. Weeks, InitPercent, InitWeeks)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Multiple Linear Regression \nX = train[['Weeks', 'InitFVC','InitWeeks']]\nY = train['FVC']\n\nregr = linear_model.LinearRegression()\nregr.fit(X, Y)\n\nprint('Intercept: \\n', regr.intercept_)\nprint('Coefficients: \\n', regr.coef_)\n\nmodel = sm.OLS(Y, X).fit()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. MLR (FVC vs. Weeks, Age, InitWeeks, InitFVC)\n* Incorporation of age --> p-value of predictor < 0.05. \n* Age could have an effect on **rate of decline**. ","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Multiple Linear Regression \nX = train[['Weeks', 'InitFVC', 'InitWeeks', 'Age']]\nY = train['FVC']\n\nregr = linear_model.LinearRegression()\nregr.fit(X, Y)\n\nprint('Intercept: \\n', regr.intercept_)\nprint('Coefficients: \\n', regr.coef_)\n\nmodel = sm.OLS(Y, X).fit()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"X_test = test_df[['Weeks', 'FVC', 'InitWeeks', 'Age']]\nY_test = test_df['FVC']\n\nY_pred = regr.predict(X_test)\n# print('Predicted FVCs: \\n', Y_pred)\n# print('Actual FVCs: \\n', Y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. MLR (FVC vs. Weeks, Age, Sex, InitWeeks, InitFVC)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Multiple Linear Regression \nX = train_df[['Weeks', 'InitFVC', 'InitWeeks', 'Age', 'SmokingStatus_Currently smokes']]\nY = train_df['FVC']\n\nregr = linear_model.LinearRegression()\nregr.fit(X, Y)\n\nprint('Intercept: \\n', regr.intercept_)\nprint('Coefficients: \\n', regr.coef_)\n\nmodel = sm.OLS(Y, X).fit()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Sample Submission\n\n* **Confidence** - Redefined as **UNCERTAINTY/STANDARD DEVIATION**\n* TODO: Implement confidence optimization function based on predictions. \n* Current optimal confidence estimate ~ **244** based on graph from https://www.kaggle.com/rohanrao/osic-understanding-laplace-log-likelihood/data?select=train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use df.iloc[idx] to access specific row \ndata = []\nfor i in range(test_df.shape[0]):\n    new_cols = ['Patient_Week', 'FVC', 'Confidence']\n    new_vals = [test_df.iloc[i]['Patient']+\"_\"+str(test_df.iloc[i]['Weeks']), Y_pred[i], 244]\n    data.append(dict(zip(new_cols, new_vals)))\nsubmission = pd.DataFrame(data)\nsubmission.head(95)\n\n# create function to compare dif between actual and pred fvc for given week ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}