{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nsns.set_palette('deep')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsubmission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.lineplot(x=\"Weeks\", y=\"FVC\",\n             hue=\"Sex\", style=\"SmokingStatus\",\n             data=train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No clear trend visible"},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric(trueFVC, predFVC, predSTD):\n    clipSTD = np.clip(predSTD, 70, 9e9)\n    deltaFVC = np.clip(np.abs(trueFVC - predFVC), 0, 1000)\n    return np.mean(-1*(np.sqrt(2)*deltaFVC/clipSTD) - np.log(np.sqrt(2) * clipSTD))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make train like test\ntrain = pd.merge(train, train.groupby('Patient').nth(0)['Weeks'].rename(\"Weeks_init\"), on='Patient', how='left')\ntrain = pd.merge(train, train.groupby('Patient').nth(0)['FVC'].rename(\"FVC_init\"), on='Patient', how='left')\ntrain = pd.merge(train, train.groupby('Patient').nth(0)['Percent'].rename(\"Percent_init\"), on='Patient', how='left')\ntrain[\"Confidence\"] = 100\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Just using the first FVC value\nprint('Metric:', metric(trueFVC=train.groupby('Patient').tail(3)['FVC'].values, predFVC=train.groupby('Patient').tail(3)['FVC_init'].values, predSTD=train.groupby('Patient').tail(3)['Confidence'].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"FVC_norm\"] = train[\"FVC\"] / train[\"FVC_init\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\n\nsns.lineplot(x=\"Weeks\", y=\"FVC_norm\",\n             hue=\"Sex\", style=\"SmokingStatus\",\n             data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\nsns.lmplot(x=\"Weeks\", y=\"FVC_norm\", hue=\"Sex\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\nsns.lmplot(x=\"Weeks\", y=\"FVC_norm\", hue=\"SmokingStatus\", data=train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"normalizing FVC helps"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Weeks should be normalized to start at zero!\ntrain[\"Weeks_norm\"] = train[\"Weeks\"] - train[\"Weeks_init\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\n\nsns.lineplot(x=\"Weeks_norm\", y=\"FVC_norm\",\n             hue=\"Sex\", style=\"SmokingStatus\",\n             data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\nsns.lmplot(x=\"Weeks_norm\", y=\"FVC_norm\", hue=\"Sex\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\nsns.lmplot(x=\"Weeks_norm\", y=\"FVC_norm\", hue=\"SmokingStatus\", data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\nsns.lmplot(x=\"Weeks_norm\", y=\"FVC_norm\", hue=\"SmokingStatus\", col=\"Sex\", data=train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Normalizing also by starting week, looks even better"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import datasets, linear_model\nfrom sklearn.metrics import mean_absolute_error\n\n# Create linear regression object\nregr = linear_model.LinearRegression()\n\nX = train[\"Weeks_norm\"].values\nY = train[\"FVC_norm\"].values\n\n# Train the model using the training sets\nregr.fit(X.reshape(-1, 1), Y)\n\n# The coefficients\nprint('Coefficients: ', regr.coef_)\nprint('Intercept: ', regr.intercept_)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's use that slope for our submission and scale with the FVC initial value!"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0]) \nsubmission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_new = pd.merge(submission, test[['Patient','Percent','Age','Sex','SmokingStatus', 'FVC', 'Weeks']], on='Patient', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_new = test_new.rename(columns={\"Weeks_x\": \"Weeks\", \"Weeks_y\": \"Weeks_init\"})\ntest_new = test_new.rename(columns={\"FVC_x\": \"FVC_2000\", \"FVC_y\": \"FVC_init\"})\ntest_new[\"Weeks_norm\"] = test_new[\"Weeks\"] - test_new[\"Weeks_init\"]\ntest_new.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_new[\"FVC_pred_linear\"] = (1 - 0.00139284 * test_new[\"Weeks_norm\"]) * test_new[\"FVC_init\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_new[\"Confidence\"] = 285  # hyperparameter chosen by best value from train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_new = test_new.rename(columns={\"FVC_pred_linear\": \"FVC\"})\ntest_new[['Patient_Week','FVC','Confidence']].to_csv('submission.csv', index=False)\ntest_new[['Patient_Week','FVC','Confidence']].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}