{"cells":[{"metadata":{},"cell_type":"markdown","source":"Late to the party but never too late. This is my straight to the point, basic EDA for getting my bearings as a newcomer.\n\nPS: When you're ready to knuckle down and get serious, check out this beast of an EDA by another Kaggler https://www.kaggle.com/piantic/osic-pulmonary-fibrosis-progression-basic-eda","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.colors as mcolors\nimport numpy as np\n\nENV = 'kaggle'\n\nif ENV == 'kaggle':\n    DATA_ROOT_PATH = '../input/osic-pulmonary-fibrosis-progression'\n    TRAIN_CSV_PATH = os.path.join(DATA_ROOT_PATH, 'train.csv')\n    TEST_CSV_PATH = os.path.join(DATA_ROOT_PATH, 'test.csv')\n    SUBMISSION_CSV_PATH = os.path.join(DATA_ROOT_PATH, 'sample_submission.csv')\nelse:\n    raise UserWarning(\"Please choose a valid ENV\")\n    \nfont = {'family' : 'normal',\n        'size'   : 16}\nplt.rc('font', **font)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Basic EDA of train csv","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_CSV_PATH).sort_values(['Patient', 'Weeks'])\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Let's look at the first-order breakdowns of patients by parameter","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"{len(train_df)} entries in training data\")\nprint(f\"{len(train_df['Patient'].unique())} unique patients\")\n\nfig, axes = plt.subplots(3, 2, figsize=(20,14))\n\naxes = axes.flatten()\n\ntrain_df[['Patient', 'SmokingStatus']].drop_duplicates().groupby(\n    ['SmokingStatus']).count().plot.bar(title='Patients segmented by smoking status', ax=axes[0]);\n\ntrain_df[['Patient', 'Sex']].drop_duplicates().groupby(\n    ['Sex']).count().plot.bar(title='Patients segmented by sex', ax=axes[1]);\n\nbin_size = 10\nage_bins = pd.cut(train_df[['Patient', 'Age']].drop_duplicates()['Age'],\n                  range(min(train_df['Age'] // bin_size)*bin_size, max(train_df['Age'] // bin_size)*bin_size + bin_size*2, bin_size))\ntrain_df[['Patient', 'Age']].drop_duplicates().groupby(\n    age_bins)['Age'].count().plot.bar(title='Patients segmented by age', ax=axes[2]);\n\nbin_size = 500\nfirst_fvc_df = train_df[['Patient', 'FVC', 'Percent']].drop_duplicates(subset=['Patient'])\nfvc_bins = pd.cut(first_fvc_df['FVC'],\n                  range(min(train_df['FVC'] // bin_size)*bin_size, max(train_df['FVC'] // bin_size)*bin_size + bin_size*2, bin_size))\nfirst_fvc_df.groupby(fvc_bins)['FVC'].count().plot.bar(title='Patients segmented by first FVC reading (mL)', ax=axes[3]);\n\nbin_size = 10\npct_bins = pd.cut(first_fvc_df['Percent'],\n                  range(int(min(train_df['Percent'] // bin_size)*bin_size), int(max(train_df['Percent'] // bin_size)*bin_size + bin_size*2), bin_size))\nfirst_fvc_df.groupby(pct_bins)['Percent'].count().plot.bar(title='Patients segmented by first FVC reading (percent)', ax=axes[4]);\n\ntrain_df.groupby('Patient').count().groupby('FVC').count()['Sex'].plot.bar(title='Patients segmented by number of FVC measurements', ax=axes[5]);\naxes[4].set_xlabel('# FVC measurements')\n\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### It looks like the smoking status and age of patients doesn't change in the dataset. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patients = train_df['Patient'].unique()\n\nfound_change = False\nfor patient in patients:\n    if len(train_df[train_df['Patient'] == patient]['SmokingStatus'].unique()) > 1:\n        found_change = True\n        \nprint(found_change)\n\nfound_change = False\nfor patient in patients:\n    if len(train_df[train_df['Patient'] == patient]['Age'].unique()) > 1:\n        found_change = True\n        \nprint(found_change)\n\n# why not check while I'm at it...\nfound_change = False\nfor patient in patients:\n    if len(train_df[train_df['Patient'] == patient]['Sex'].unique()) > 1:\n        found_change = True\n        \nprint(found_change)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Let's look at some individual patients and their FVC over time","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_cols = 3\nn_rows = 5\nfig, axes = plt.subplots(n_rows, n_cols, figsize=(6*n_cols, 3*n_rows))\naxes = axes.flatten()\n\nfor i, patient in enumerate(np.random.choice(patients, min(n_cols*n_rows, len(patients)), replace=False)):\n    patient_df = train_df[train_df['Patient'] == patient]\n    sex = patient_df['Sex'].unique()[0]\n    age = patient_df['Age'].unique()[0]\n    smoking_status = patient_df['SmokingStatus'].unique()[0]\n    patient_df.plot('Weeks', 'FVC', ax=axes[i], title=f\"{sex}, aged {age}, {smoking_status}\", lw=2, style='o-')\n    axes[i].grid()\n\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Do patients with similar stats have similar FVC progression?","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### CHAGE FILTER PARAMS HERE ###\nsex = 'Male'\nage_bracket = (50, 70)\nsmoking_status = ['Ex-smoker', 'Currently smokes', 'Never smoked'][1]\n###\n\ntrain_df_filt = train_df[train_df['Sex'] == sex]\ntrain_df_filt = train_df_filt[train_df_filt['Age'] >= age_bracket[0]]\ntrain_df_filt = train_df_filt[train_df_filt['Age'] < age_bracket[1]]\ntrain_df_filt = train_df_filt[train_df_filt['SmokingStatus'] == smoking_status]\npatients = train_df_filt['Patient'].unique()\n\nn_cols = 3\nn_rows = min(5, -(- len(patients) // n_cols))\nfig, axes = plt.subplots(n_rows, n_cols, figsize=(6*n_cols, 3*n_rows))\naxes = axes.flatten()\n\nfor i, patient in enumerate(np.random.choice(patients, min(n_cols*n_rows, len(patients)), replace=False)):\n    patient_df = train_df_filt[train_df_filt['Patient'] == patient]\n    sex = patient_df['Sex'].unique()[0]\n    age = patient_df['Age'].unique()[0]\n    smoking_status = patient_df['SmokingStatus'].unique()[0]\n    patient_df.plot('Weeks', 'FVC', ax=axes[i], title=f\"{sex}, aged {age}, {smoking_status}\", lw=2, style='o-')\n    axes[i].grid()\n\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Is there any obvious decline from first FVC measurement to last?\n\nYep.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"patients = train_df['Patient'].unique()\n\nfirst = []\nlast = []\n\nfor patient in patients:\n    patient_df = train_df[train_df['Patient'] == patient]\n    first.append(list(patient_df['Percent'])[0])\n    last.append(list(patient_df['Percent'])[-1])\n\nfig, ax = plt.subplots()\nax.scatter(first, last)\nm, b = np.polyfit(first, last, 1)\nax.plot(first, m*np.array(first) + b, color='r')\nax.plot(first, first, color='lightgray')\nax.set_xlabel('First FVC result (%)')\nax.set_ylabel('Last FVC result (%)')\nax.set_title(f\"last = {m:.2f}first + {b:.2f}\")\nax.grid()\nax.legend(['Fit', 'x=x']);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### And if we narrow down on a certain subset of patients?\n\nYep.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### CHAGE FILTER PARAMS HERE ###\nsex = 'Male'\nage_bracket = (50, 70)\nsmoking_status = ['Ex-smoker', 'Currently smokes', 'Never smoked'][0]\n###\n\ntrain_df_filt = train_df[train_df['Sex'] == sex]\ntrain_df_filt = train_df_filt[train_df_filt['Age'] >= age_bracket[0]]\ntrain_df_filt = train_df_filt[train_df_filt['Age'] < age_bracket[1]]\ntrain_df_filt = train_df_filt[train_df_filt['SmokingStatus'] == smoking_status]\npatients = train_df_filt['Patient'].unique()\n\nfirst = []\nlast = []\n\nfor patient in patients:\n    patient_df = train_df[train_df['Patient'] == patient]\n    first.append(list(patient_df['Percent'])[0])\n    last.append(list(patient_df['Percent'])[-1])\n\nfig, ax = plt.subplots()\nax.scatter(first, last)\nm, b = np.polyfit(first, last, 1)\nax.plot(first, m*np.array(first) + b, color='r')\nax.plot(first, first, color='lightgray')\nax.set_xlabel('First FVC result (%)')\nax.set_ylabel('Last FVC result (%)');\nax.set_title(f\"last = {m:.2f}first + {b:.2f}\")\nax.grid()\nax.legend(['Fit', 'x=x'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Is there a clear relationship between FVC and Percent, given the other parameters?\n\nApparently not. There are some hidden parameters that determine the relationship between FVC and Percent, as can be seen when we try narrowing down on particular subsegements. The intragroup variation within the ages is just as much as the itergroup variation, meaning that hidden parameters seem to have a much bigger affect on FVC/Percent than age.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"### CHAGE FILTER PARAMS HERE ###\nsex = 'Male'\nage_brackets = [(50, 60), (60, 70), (70, 80)]\nsmoking_status = ['Ex-smoker', 'Currently smokes', 'Never smoked'][0]\n###\n\ntrain_df_filt = train_df[train_df['Sex'] == sex]\ntrain_df_filt = train_df_filt[train_df_filt['SmokingStatus'] == smoking_status]\n\n# PLOT INDIVIDUALLY\nn_ages = len(age_brackets)\nn_cols = 3\nn_rows = min(5, -(- n_ages // n_cols))\nfig, axes = plt.subplots(n_rows, n_cols, figsize=(6*n_cols, 5*n_rows))\naxes = axes.flatten()\n\nfor i, age_bracket in enumerate(age_brackets):\n    age_df = train_df_filt[train_df_filt['Age'] < age_bracket[1]]\n    age_df = age_df[age_df['Age'] >= age_bracket[0]]\n    if not len(age_df):\n        continue\n    m, b = np.polyfit(age_df['Percent'], age_df['FVC'], 1)\n    age_df.plot.scatter('Percent', 'FVC', ax=axes[i], title=f\"Age bracket: {age_bracket}\", lw=2, style='o-')\n    axes[i].plot(np.array(age_df['Percent']), m*np.array(age_df['Percent']) + b, color='r')\n    axes[i].grid()\n    \nplt.tight_layout();\n\n# PLOT TOGETHER\nfig, ax = plt.subplots(1,1,figsize=(10,8))\n\n# color selection\ncolors = np.random.choice(list(mcolors.TABLEAU_COLORS.keys()),\n                          size=len(age_brackets),\n                          replace=False)\n\nfor i, age_bracket in enumerate(age_brackets):\n    age_df = train_df_filt[train_df_filt['Age'] < age_bracket[1]]\n    age_df = age_df[age_df['Age'] >= age_bracket[0]]\n    fvc = np.array(age_df['FVC'])\n    if not len(fvc):\n        continue\n    percent = np.array(age_df['Percent'])\n    ax.scatter(fvc, percent, color=colors[(age-age_bracket[0]) % len(colors)])\n    m, b = np.polyfit(fvc, percent, 1)\n    ax.plot(fvc, m*fvc + b, color=colors[(age-age_bracket[0]) % len(colors)])\n    \nax.set_title('All plotted together for comparison')\nax.set_xlabel('FVC')\nax.set_ylabel('Percent')\nax.grid()\nax.legend(age_brackets);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Just to be sure, let's check the other parameters... It looks like at least the Sex seems to play an important rolw in the FVC/Percent ratio.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(20,5))\nax = ax.flatten()\n\ndef plot(param, ax, designations, colors):\n    for color, designation in zip(colors, designations):\n        train_df[train_df[param] == designation].plot.scatter('FVC', 'Percent', ax=ax, color=color, marker='.')\n        fvc = train_df[train_df[param] == designation]['FVC']\n        percent = train_df[train_df[param] == designation]['Percent']\n        m, b = np.polyfit(fvc, percent, 1)\n        ax.plot(fvc, m*fvc + b, color=color, linewidth=2)\n        ax.legend(designations, loc='lower right')\n\nplot('Sex', ax[0], ['Male', 'Female'], ['green', 'blue'])\nplot('SmokingStatus', ax[1], ['Ex-smoker', 'Currently smokes', 'Never smoked'], ['green', 'blue', 'purple'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}