{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n\nfrom scipy import stats\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ndf_test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Training Set Shape = {df_train.shape} - Patients = {df_train[\"Patient\"].nunique()}')\nprint(f'Test Set Shape = {df_test.shape} - Patients = {df_test[\"Patient\"].nunique()}')\nprint(f'The names of the features are {list(df_train.columns)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FVCMeasurements_train = df_train.rename(columns={'Weeks': 'FVCMeasurements'}).groupby('Patient').agg('count')['FVCMeasurements'].value_counts()\nprint(f'Training Set FVC Measurements Per Patient \\n{(\"-\") * 41}\\n{FVCMeasurements_train}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv' )\ndf_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def laplace_log_likelihood(y_true, y_pred, sigma):\n    sigma_clipped = np.maximum(sigma, 70)\n    delta_clipped = np.minimum(np.abs(y_true - y_pred), 1000)\n    metric = - np.sqrt(2) * delta_clipped / sigma_clipped - np.log(np.sqrt(2) * sigma_clipped)\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"laplace_log_likelihood(df_train['FVC'], df_train['FVC'], 70)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'FVC Statistical Properties \\n{\"-\" * 26}\\n')\nprint(f'Mean: {df_train[\"FVC\"].mean():.6}')\nprint(f'Median: {df_train[\"FVC\"].median():.6}')\nprint(f'Std: {df_train[\"FVC\"].std():.6}')\nprint(f'Min: {df_train[\"FVC\"].min()}')\nprint(f'25%: {df_train[\"FVC\"].quantile(0.25)}')\nprint(f'50%: {df_train[\"FVC\"].quantile(0.5)}')\nprint(f'75%: {df_train[\"FVC\"].quantile(0.75)}')\nprint(f'Max: {df_train[\"FVC\"].max()}')\nprint(f'Skew: {df_train[\"FVC\"].skew():.6}')\nprint(f'Kurtosis: {df_train[\"FVC\"].kurtosis():.6}')\nmissing_values= df_train[df_train[\"FVC\"].isnull()].shape[0]\ntraining_samples= df_train.shape[0]\nprint(f'Missing Values: {missing_values}/{training_samples} ({missing_values * 100 / training_samples:.4}%)')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(figsize=(18, 6))\nsns.distplot(df_train['FVC'], label='FVC').set_title('FVC Distribution in Training Set', pad=15, size =18)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for patient, df in list(df_train.groupby('Patient')):\n    print(f'Patient: {patient} FVC Statistical Properties\\n{\"-\" * 61}')\n    print(f'Mean: {df[\"FVC\"].mean():.6}')\n    print(f'Median: {df[\"FVC\"].median():.6}')\n    print(f'Std: {df[\"FVC\"].std():.6}')\n    print(f'Min: {df[\"FVC\"].min()}')\n    print(f'Max: {df[\"FVC\"].max()}')\n    print(f'Skew: {df[\"FVC\"].skew():.6}')\n    print(f'Kurtosis: {df[\"FVC\"].kurtosis():.6}')\n    print(f'\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_fvc(df, patient): \n    df[['Weeks', 'FVC']].set_index('Weeks').plot(figsize=(30, 6), label='_nolegend_')\n    plt.tick_params(axis='x', labelsize=20)\n    plt.tick_params(axis='y', labelsize=20)\n    plt.xlabel('')\n    plt.ylabel('')\n    plt.title(f'Patient: {patient} - {df[\"Age\"].tolist()[0]} years old - {df[\"Sex\"].tolist()[0]} - {df[\"SmokingStatus\"].tolist()[0]} ({len(df)} Measurements in {(df[\"Weeks\"].max() - df[\"Weeks\"].min())} Weeks Period)', size=25, pad=25)\n    print(f'\\n')\n    plt.legend().set_visible(False)\n    plt.show()\n      \nfor patient, df in list(df_train.groupby('Patient')):  \n    plot_fvc(df, patient)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.pairplot(df_train[['FVC', 'Weeks', 'Percent', 'Age']], aspect=1.4, height=5, diag_kind='kde', kind='reg')\n\ng.axes[3, 0].set_xlabel('FVC', fontsize=25)\ng.axes[3, 1].set_xlabel('Weeks', fontsize=25)\ng.axes[3, 2].set_xlabel('Percent', fontsize=25)\ng.axes[3, 3].set_xlabel('Age', fontsize=25)\ng.axes[0, 0].set_ylabel('FVC', fontsize=25)\ng.axes[1, 0].set_ylabel('Weeks', fontsize=25)\ng.axes[2, 0].set_ylabel('Percent', fontsize=25)\ng.axes[3, 0].set_ylabel('Age', fontsize=25)\n\ng.axes[3, 0].tick_params(axis='x', labelsize=20)\ng.axes[3, 1].tick_params(axis='x', labelsize=20)\ng.axes[3, 2].tick_params(axis='x', labelsize=20)\ng.axes[3, 3].tick_params(axis='x', labelsize=20)\ng.axes[0, 0].tick_params(axis='y', labelsize=20)\ng.axes[1, 0].tick_params(axis='y', labelsize=20)\ng.axes[2, 0].tick_params(axis='y', labelsize=20)\ng.axes[3, 0].tick_params(axis='y', labelsize=20)\n\ng.fig.suptitle('Pair Plots for Continuous features', fontsize=35, y=1.08)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(8, 8), dpi=100)\n\nsns.heatmap(df_train.corr(), annot=True, square=True, cmap='BuPu', annot_kws={'size': 10}, fmt='.2f', linewidths=.5)   \n\nplt.tick_params(axis='x', labelsize=10, rotation=0)\nplt.tick_params(axis='y', labelsize=10, rotation=0)\nplt.title('Correlation Matrix for Continuous Data', size=18, pad=18)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(ncols=2,figsize=(18, 6))\n\nsns.barplot(x=df_train.groupby('Patient')['Sex'].first().value_counts().index, y=df_train.groupby('Patient')['Sex'].first().value_counts(), ax=axes[0], color=\"skyblue\")\npercentages0 = [(count / df_train.groupby('Patient')['Sex'].first().value_counts().sum() * 100).round(2) for count in df_train.groupby('Patient')['Sex'].first().value_counts()]\n\naxes[0].set_ylabel('')\naxes[0].set_xticks(np.arange(2), [f'Male (%{percentages0[0]})', f'Female (%{percentages0[1]})'])\naxes[0].tick_params(axis='x', labelsize=12)\naxes[0].tick_params(axis='y', labelsize=12)\naxes[0].set_title('Sex Histogram', size=15, pad=15)\n\n\nsns.barplot(x=df_train.groupby('Patient')['SmokingStatus'].first().value_counts().index, y=df_train.groupby('Patient')['SmokingStatus'].first().value_counts(),ax=axes[1], color=\"violet\")\npercentages1 = [(count / df_train.groupby('Patient')['SmokingStatus'].first().value_counts().sum() * 100).round(2) for count in df_train.groupby('Patient')['SmokingStatus'].first().value_counts()]\n\naxes[1].set_ylabel('')\naxes[1].set_xticks(np.arange(3), [f'Ex-smoker (%{percentages1[0]})', f'Never smoked (%{percentages1[1]})', f'Currently Smokes (%{percentages1[2]})'])\naxes[1].tick_params(axis='x', labelsize=12)\naxes[1].tick_params(axis='y', labelsize=12)\naxes[1].set_title('Smoking Status histogram', size=15, pad=15)\n\nplt.show()\nprint(f'Sex\\n{\"-\" * 40}')\nprint(f'Number of males:')\nprint(df_train.groupby('Patient')['Sex'].first().value_counts()[0])\nprint(f'Number of females:')\nprint(df_train.groupby('Patient')['Sex'].first().value_counts()[1])\nprint(f'Male (%{percentages0[0]})', f'Female (%{percentages0[1]})')\nprint(f'\\n')\nprint(f'Smoking Status\\n{\"-\" * 40}')\nprint(f'Ex-smoker:')\nprint(df_train.groupby('Patient')['SmokingStatus'].first().value_counts()[0])\nprint(f'Never Smoked:')\nprint(df_train.groupby('Patient')['SmokingStatus'].first().value_counts()[1])\nprint(f'Currently Smokes:')\nprint(df_train.groupby('Patient')['SmokingStatus'].first().value_counts()[2])\nprint(f'Ex-smoker (%{percentages1[0]})', f'Never smoked (%{percentages1[1]})', f'Currently Smokes (%{percentages1[2]})')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.pairplot(df_train[['FVC', 'Weeks', 'Percent', 'Age', 'Sex']], hue='Sex', aspect=1.4, height=5, diag_kind='kde', kind='reg')\ng.axes[3, 0].set_xlabel('FVC', fontsize=25)\ng.axes[3, 1].set_xlabel('Weeks', fontsize=25)\ng.axes[3, 2].set_xlabel('Percent', fontsize=25)\ng.axes[3, 3].set_xlabel('Age', fontsize=25)\ng.axes[0, 0].set_ylabel('FVC', fontsize=25)\ng.axes[1, 0].set_ylabel('Weeks', fontsize=25)\ng.axes[2, 0].set_ylabel('Percent', fontsize=25)\ng.axes[3, 0].set_ylabel('Age', fontsize=25)\ng.axes[3, 0].tick_params(axis='x', labelsize=20)\ng.axes[3, 1].tick_params(axis='x', labelsize=20)\ng.axes[3, 2].tick_params(axis='x', labelsize=20)\ng.axes[3, 3].tick_params(axis='x', labelsize=20)\ng.axes[0, 0].tick_params(axis='y', labelsize=20)\ng.axes[1, 0].tick_params(axis='y', labelsize=20)\ng.axes[2, 0].tick_params(axis='y', labelsize=20)\ng.axes[3, 0].tick_params(axis='y', labelsize=20)\nplt.legend(prop={'size': 20})\ng._legend.remove()\ng.fig.suptitle('Pair Plots Between Sex Groups', fontsize=40, y=1.1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.pairplot(df_train[['FVC', 'Weeks', 'Percent', 'Age', 'SmokingStatus']], hue='SmokingStatus', aspect=1.4, height=5, diag_kind='kde', kind='reg')\ng.axes[3, 0].set_xlabel('FVC', fontsize=25)\ng.axes[3, 1].set_xlabel('Weeks', fontsize=25)\ng.axes[3, 2].set_xlabel('Percent', fontsize=25)\ng.axes[3, 3].set_xlabel('Age', fontsize=25)\ng.axes[0, 0].set_ylabel('FVC', fontsize=25)\ng.axes[1, 0].set_ylabel('Weeks', fontsize=25)\ng.axes[2, 0].set_ylabel('Percent', fontsize=25)\ng.axes[3, 0].set_ylabel('Age', fontsize=25)\ng.axes[3, 0].tick_params(axis='x', labelsize=20)\ng.axes[3, 1].tick_params(axis='x', labelsize=20)\ng.axes[3, 2].tick_params(axis='x', labelsize=20)\ng.axes[3, 3].tick_params(axis='x', labelsize=20)\ng.axes[0, 0].tick_params(axis='y', labelsize=20)\ng.axes[1, 0].tick_params(axis='y', labelsize=20)\ng.axes[2, 0].tick_params(axis='y', labelsize=20)\ng.axes[3, 0].tick_params(axis='y', labelsize=20)\nplt.legend(prop={'size': 20})\ng._legend.remove()\ng.fig.suptitle('Pair Plots Between SmokingStatus Groups', fontsize=40, y=1.1)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}