{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is my first attempt of an EDA Notebook for Kaggle, this was heavily ensipired by the work of [Kelli Belchers EDA](https://www.kaggle.com/code/kellibelcher/amex-default-prediction-eda-lgbm-baseline)","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport plotly.express as px\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:28:18.643005Z","iopub.execute_input":"2022-07-27T14:28:18.643598Z","iopub.status.idle":"2022-07-27T14:28:21.281429Z","shell.execute_reply.started":"2022-07-27T14:28:18.643466Z","shell.execute_reply":"2022-07-27T14:28:21.279894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the compressed Data of the Amex Dataset #","metadata":{}},{"cell_type":"code","source":"df = pd.read_feather('../input/amexfeather/train_data.ftr')\ndf = df.groupby('customer_ID').tail(1).set_index('customer_ID')\npal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:28:21.284721Z","iopub.execute_input":"2022-07-27T14:28:21.285902Z","iopub.status.idle":"2022-07-27T14:28:42.576885Z","shell.execute_reply.started":"2022-07-27T14:28:21.285836Z","shell.execute_reply":"2022-07-27T14:28:42.575599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Brief overview of the DataFrame ##","metadata":{}},{"cell_type":"markdown","source":"1. Display DataFrame Head\n2. Check number rows and feautres, type of Data\n3. Check Distribution of paid and defaulted Credits","metadata":{}},{"cell_type":"markdown","source":"### 1. Display DataFrame Head ###","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:28:42.578622Z","iopub.execute_input":"2022-07-27T14:28:42.579013Z","iopub.status.idle":"2022-07-27T14:28:42.615493Z","shell.execute_reply.started":"2022-07-27T14:28:42.578980Z","shell.execute_reply":"2022-07-27T14:28:42.614091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Describe DataFrame ###","metadata":{}},{"cell_type":"code","source":"print(\"The DataFrame consists of {} rows and {} features\".format(df.shape[0],df.shape[1]))\ndf.describe()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:28:42.619139Z","iopub.execute_input":"2022-07-27T14:28:42.619693Z","iopub.status.idle":"2022-07-27T14:28:58.139365Z","shell.execute_reply.started":"2022-07-27T14:28:42.619629Z","shell.execute_reply":"2022-07-27T14:28:58.137922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Check Distribution of target variable ###","metadata":{}},{"cell_type":"code","source":"# plot percentage of distribution count for target variable\nfig = px.pie(df, names='target', title=\"Distribution of defaulted Customers (1) and paying Customers(0)\")\nfig.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:28:58.141026Z","iopub.execute_input":"2022-07-27T14:28:58.141388Z","iopub.status.idle":"2022-07-27T14:28:59.554779Z","shell.execute_reply.started":"2022-07-27T14:28:58.141355Z","shell.execute_reply":"2022-07-27T14:28:59.553913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean DataFrame based on NaN Entries ##","metadata":{}},{"cell_type":"markdown","source":"### Drop Columns ###","metadata":{}},{"cell_type":"code","source":"remove_cols = [col for col in df.columns if df[col].isnull().all() or df[col].isnull().sum()> df.shape[0]/2]\nstring_cols = [col for col in df.columns if df[col].dtype == \"object\"]\ncategorical_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68', 'target']\nremove_cols_2 = [\"D_116\", \"S_2\"]\nprint(\"the columns {} should be dropped due to more than 50% of NaN entries\".format(remove_cols))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:28:59.556062Z","iopub.execute_input":"2022-07-27T14:28:59.556930Z","iopub.status.idle":"2022-07-27T14:29:00.417921Z","shell.execute_reply.started":"2022-07-27T14:28:59.556892Z","shell.execute_reply":"2022-07-27T14:29:00.416827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop(axis = 1, columns=remove_cols, inplace=True,)\nprint(\"new shape of DataFrame after dropping columns :\", df.shape)\n### col_types ###\ncategorical_cols = [col for col in categorical_cols if not col in remove_cols]\ndelinquency_cols = [col for col in df.columns if \"D_\" in col and col not in categorical_cols]\npayment_cols = [col for col in df.columns if \"P_\" in col and col not in categorical_cols]\nspend_cols = [col for col in df.columns if \"S_\" in col and col not in categorical_cols]\nrisk_cols = [col for col in df.columns if \"R_\" in col and col not in categorical_cols]\nbalance_cols = [col for col in df.columns if \"B_\" in col and col not in categorical_cols]\nprint(\"There are {} categorical columns, {} delinquency columns, {} payment columns, {} spent columns, {} risk columns and {} balance columns\".format(len(categorical_cols),\n                                                                                                                                  len(delinquency_cols),\n                                                                                                                                  len(payment_cols),\n                                                                                                                                  len(spend_cols),\n                                                                                                                                  len(risk_cols),\n                                                                                                                                  len(balance_cols)))\ndelinquency_cols.append(\"target\") #adds the target variable to the delinquency column for later visulization\npayment_cols.append(\"target\") #adds the target variable to the payment column for later visulization\nspend_cols.append(\"target\") #adds the target variable to the spending column for later visulization\nrisk_cols.append(\"target\") #adds the target variable to the risk column for later visulization\nbalance_cols.append(\"target\") #adds the target variable to the balance column for later visulization","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:29:00.419533Z","iopub.execute_input":"2022-07-27T14:29:00.419853Z","iopub.status.idle":"2022-07-27T14:29:00.764039Z","shell.execute_reply.started":"2022-07-27T14:29:00.419823Z","shell.execute_reply":"2022-07-27T14:29:00.763055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Drop Rows ###","metadata":{}},{"cell_type":"code","source":"num_nan = sum(df.isnull().sum(axis=1))\nnum_entries = df.shape[0]*df.shape[1]\nplot_nan = [num_nan/num_entries, (num_entries-num_nan)/num_entries]\nnan_labels = [\"NaN Entries\", \"Valid Entries\"]\nplt.pie(plot_nan, labels = nan_labels, colors = color, autopct='%.0f%%')\nplt.title(\"Share of NaN Entries\")\nplt.show()\n\nprint(\"A total of {} NaN Entries are present in the DataFrame\".format(num_nan))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:29:00.765312Z","iopub.execute_input":"2022-07-27T14:29:00.766277Z","iopub.status.idle":"2022-07-27T14:29:01.392553Z","shell.execute_reply.started":"2022-07-27T14:29:00.766238Z","shell.execute_reply":"2022-07-27T14:29:01.390966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check Number of NaN Entries for each row, if over a given threshold, row is dropped from the DataFrame.<br>\nGiven the pie-Chart from above, a average of 2% of NaN Entries is to be expected.<br>\nCurrently, Threshold is set to 5% which drops roughly 0.03% of the rows","metadata":{}},{"cell_type":"code","source":"count_nan = df.isnull().sum(axis=1)\nremove_rows = [row for idx, row in enumerate(df.index) if count_nan[idx] > 9]\ndf.drop(axis = 0, index=remove_rows, inplace=True,)\nprint(\"A total fo {} rows where dropped due to more than 5% NaN entries\".format(len(remove_rows)))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:29:01.394882Z","iopub.execute_input":"2022-07-27T14:29:01.396672Z","iopub.status.idle":"2022-07-27T14:29:03.884438Z","shell.execute_reply.started":"2022-07-27T14:29:01.396578Z","shell.execute_reply":"2022-07-27T14:29:03.883181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA #","metadata":{}},{"cell_type":"markdown","source":"## Explore the Categorical Variables ##","metadata":{}},{"cell_type":"code","source":"pal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']\nrow_s = int(len(categorical_cols)**(1/2))\nif row_s**2 < len(categorical_cols):\n    col_s = row_s +1\nelse:\n    col_s = row_s\nfig, axs = plt.subplots(row_s, col_s, figsize=(20,20))\nfig.suptitle('Categorical Variables',fontsize=16)\ncolumn = list(range(col_s))*row_s\nrow=0\nfor i, col in enumerate(categorical_cols[:-1]):\n    if (i!=0)&(i%(row_s+1)==0):\n        row+=1\n    sns.histplot(x=col, hue='target', palette=pal[::-1], hue_order=[1,0], \n        label=['Default','Paid'], data=df, multiple=\"stack\", \n        ax=axs[row,column[i]], legend=False)\n    # axs[row,column[i]].tick_params(left=False,bottom=False)\n    axs[row,column[i]].set(title='\\n\\n{}'.format(col), xlabel='', ylabel=('Count' if i%4==0 else ''))\nfor i in range(2,4):\n    axs[2,i].set_visible(False)\nhandles, _ = axs[0,0].get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.25, 0.95))\nsns.despine(bottom=True, trim=True)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:29:03.888332Z","iopub.execute_input":"2022-07-27T14:29:03.888788Z","iopub.status.idle":"2022-07-27T14:29:15.351419Z","shell.execute_reply.started":"2022-07-27T14:29:03.888745Z","shell.execute_reply":"2022-07-27T14:29:15.349964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Checking of Correlation between the categories of each categorical variable and the target","metadata":{}},{"cell_type":"code","source":"for x in categorical_cols[:-1]:\n    print('Default Correlation by:', x)\n    print(df[[x, \"target\"]].groupby(x, as_index=False).mean())\n    print(\"\\n\")","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:29:15.353698Z","iopub.execute_input":"2022-07-27T14:29:15.354411Z","iopub.status.idle":"2022-07-27T14:29:15.502901Z","shell.execute_reply.started":"2022-07-27T14:29:15.354360Z","shell.execute_reply":"2022-07-27T14:29:15.501482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the Delinquency variables","metadata":{}},{"cell_type":"code","source":"plot_df=df[delinquency_cols]\nfig, ax = plt.subplots(13,5, figsize=(16,54))\nfig.suptitle('Distribution of Delinquency Variables',fontsize=16)\nrow=0\ncol=[0,1,2,3,4]*13\nfor i, column in enumerate(plot_df.columns[:-1]):\n    if (i!=0)&(i%5==0):\n        row+=1\n    sns.kdeplot(x=column, hue='target', palette=pal[::-1], hue_order=[1,0], \n                label=['Default','Paid'], data=plot_df, \n                fill=True, linewidth=2, legend=False, ax=ax[row,col[i]])\n    ax[row,col[i]].tick_params(left=False,bottom=False)\n    ax[row,col[i]].set(title='\\n\\n{}'.format(column), xlabel='', ylabel=('Density' if i%5==0 else ''))\n# for rows in range(13,17):\n#     for i in range(5):\n#        ax[rows,i].set_visible(False)\nhandles, _ = ax[0,0].get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.18, 0.983))\nsns.despine(bottom=True, trim=True)\nplt.tight_layout(rect=[0, 0.2, 1, 0.99])","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:29:15.504964Z","iopub.execute_input":"2022-07-27T14:29:15.506261Z","iopub.status.idle":"2022-07-27T14:31:57.381142Z","shell.execute_reply.started":"2022-07-27T14:29:15.506206Z","shell.execute_reply":"2022-07-27T14:31:57.379880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(30,30))\ncorr_df = df[delinquency_cols].corr()\nmask=np.triu(np.ones_like(corr_df, dtype=bool))[1:,:-1]\ncorr_df=corr_df.iloc[1:,:-1].copy()\nfig.suptitle('Pearson Correlation of Delinquency Variables',fontsize=16)\nsns.heatmap(corr_df, mask=mask, ax=ax, annot=True, fmt=\".2f\", annot_kws={'fontsize':7,'fontweight':'bold'}, cbar=False)# , cmap=\"YlGnBu\" , annot=True","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:31:57.382771Z","iopub.execute_input":"2022-07-27T14:31:57.383121Z","iopub.status.idle":"2022-07-27T14:32:11.643067Z","shell.execute_reply.started":"2022-07-27T14:31:57.383091Z","shell.execute_reply":"2022-07-27T14:32:11.641722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for redudancy based on high correlation of deliquency variables (>.98)","metadata":{}},{"cell_type":"code","source":"High_correlation=[]\nfor row, row_val in enumerate(corr_df):\n    for col, col_val in enumerate(corr_df):\n        if row < col:\n            break\n        corr_val = corr_df.iloc[row, col]\n        if abs(corr_val) >= 0.98:\n            High_correlation.append((col_val, row_val, corr_val))\nfor entr in High_correlation:\n    print(\"{} is highly correlated with {} at an pearson correlation score of {:.3f}\". format(entr[0],entr[1], entr[2]))\ndrop_columns = [val[1] for val in High_correlation]\nprint(\"columns to be dropped due to high correlation:\", drop_columns)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:32:11.644868Z","iopub.execute_input":"2022-07-27T14:32:11.645243Z","iopub.status.idle":"2022-07-27T14:32:11.718407Z","shell.execute_reply.started":"2022-07-27T14:32:11.645209Z","shell.execute_reply":"2022-07-27T14:32:11.717092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the Spend Variables ##\n### Plotting of Variables ###","metadata":{}},{"cell_type":"code","source":"plot_df=df[spend_cols]\npal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']\nrow_num = int(len(spend_cols)/5)+1\nfig, ax = plt.subplots(row_num,5, figsize=(16,16*(row_num/5)))\nfig.suptitle('Distribution of Spend Variables',fontsize=16)\nrow=0\ncol=[0,1,2,3,4]*row_num\nfor i, column in enumerate(plot_df.columns[1:-1]):\n    if (i!=0)&(i%5==0):\n        row+=1\n    sns.kdeplot(x=column, hue='target', palette=pal[::-1], hue_order=[1,0], \n                label=['Default','Paid'], data=plot_df, \n                fill=True, linewidth=2, legend=False, ax=ax[row,col[i]])\n    ax[row,col[i]].tick_params(left=False,bottom=False)\n    ax[row,col[i]].set(title='\\n\\n{}'.format(column), xlabel='', ylabel=('Density' if i%5==0 else ''))\nfor i in range(1,5):\n    ax[row_num-1,i].set_visible(False)\nhandles, _ = ax[0,0].get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.21, 0.953))\nsns.despine(trim=True)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:32:11.720100Z","iopub.execute_input":"2022-07-27T14:32:11.720435Z","iopub.status.idle":"2022-07-27T14:33:02.130172Z","shell.execute_reply.started":"2022-07-27T14:32:11.720404Z","shell.execute_reply":"2022-07-27T14:33:02.128729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation ###","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(16,16))\ncorr_df = df[spend_cols].corr()\nmask=np.triu(np.ones_like(corr_df, dtype=bool))[1:,:-1]\ncorr_df=corr_df.iloc[1:,:-1].copy()\nfig.suptitle('Pearson Correlation of Spend Variables',fontsize=16)\nsns.heatmap(corr_df, mask=mask, ax=ax, annot=True, fmt=\".2f\", annot_kws={'fontsize':8,'fontweight':'bold'}, cbar=False)# , cmap=\"YlGnBu\" , annot=True","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:33:02.132159Z","iopub.execute_input":"2022-07-27T14:33:02.132795Z","iopub.status.idle":"2022-07-27T14:33:04.714039Z","shell.execute_reply.started":"2022-07-27T14:33:02.132748Z","shell.execute_reply":"2022-07-27T14:33:04.712792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(6,6))\nsns.scatterplot(x=\"S_22\", y=\"S_24\", data=df, hue=\"target\", palette=pal[::-1], hue_order=[1,0], \n                label=['Default','Paid'], legend=False, ax = ax)\nfig.suptitle(\"Correlation of Variable S_22 and S_24\",fontsize=16)\nhandles, _ = ax.get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.2, 0.95))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:33:04.716247Z","iopub.execute_input":"2022-07-27T14:33:04.717112Z","iopub.status.idle":"2022-07-27T14:33:13.296794Z","shell.execute_reply.started":"2022-07-27T14:33:04.717062Z","shell.execute_reply":"2022-07-27T14:33:13.295697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"High_correlation=[]\nfor row, row_val in enumerate(corr_df):\n    for col, col_val in enumerate(corr_df):\n        if row < col:\n            break\n        corr_val = corr_df.iloc[row, col]\n        if abs(corr_val) >= 0.98:\n            High_correlation.append((col_val, row_val, corr_val))\nfor entr in High_correlation:\n    print(\"{} is highly correlated with {} at a pearson correlation score of {:.3f}\". format(entr[0],entr[1], entr[2]))\ndrop_columns.extend([val[1] for val in High_correlation])\nif len(High_correlation) == 0:\n    print(\"No new columns found in spend for dropping due to high correlation\")\nelse:\n    print(\"columns to be dropped due to high correlation:\", drop_columns)\n    print(\"a total of {} columns where added\".format(len(High_correlation)))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:33:13.298220Z","iopub.execute_input":"2022-07-27T14:33:13.298919Z","iopub.status.idle":"2022-07-27T14:33:13.315254Z","shell.execute_reply.started":"2022-07-27T14:33:13.298875Z","shell.execute_reply":"2022-07-27T14:33:13.314057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the Risk variables","metadata":{}},{"cell_type":"code","source":"plot_df=df[risk_cols]\npal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']\nrow_num = int(len(risk_cols)/5)+1\nfig, ax = plt.subplots(row_num,5, figsize=(16,16*(row_num/5)))\nfig.suptitle('Distribution of Risk Variables',fontsize=16)\nrow=0\ncol=[0,1,2,3,4]*row_num\nfor i, column in enumerate(plot_df.columns[:-1]):\n    if (i!=0)&(i%5==0):\n        row+=1\n    sns.kdeplot(x=column, hue='target', palette=pal[::-1], hue_order=[1,0], \n                label=['Default','Paid'], data=plot_df, \n                fill=True, linewidth=2, legend=False, ax=ax[row,col[i]])\n    ax[row,col[i]].tick_params(left=False,bottom=False)\n    ax[row,col[i]].set(title='\\n\\n{}'.format(column), xlabel='', ylabel=('Density' if i%5==0 else ''))\nfor i in range(1,5):\n    ax[row_num-1,i].set_visible(False)\nhandles, _ = ax[0,0].get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.21, 0.953))\nsns.despine(trim=True)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:33:13.316660Z","iopub.execute_input":"2022-07-27T14:33:13.317018Z","iopub.status.idle":"2022-07-27T14:34:19.333205Z","shell.execute_reply.started":"2022-07-27T14:33:13.316985Z","shell.execute_reply":"2022-07-27T14:34:19.331568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(16,16))\ncorr_df = df[risk_cols].corr()\nmask=np.triu(np.ones_like(corr_df, dtype=bool))[1:,:-1]\ncorr_df=corr_df.iloc[1:,:-1].copy()\nfig.suptitle('Pearson Correlation of Risk Variables',fontsize=16)\nsns.heatmap(corr_df, mask=mask, ax=ax, annot=True, fmt=\".2f\", annot_kws={'fontsize':8,'fontweight':'bold'}, cbar=False)# , cmap=\"YlGnBu\" , annot=True","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:34:19.335072Z","iopub.execute_input":"2022-07-27T14:34:19.335453Z","iopub.status.idle":"2022-07-27T14:34:22.010519Z","shell.execute_reply.started":"2022-07-27T14:34:19.335415Z","shell.execute_reply":"2022-07-27T14:34:22.009154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"High_correlation=[]\nfor row, row_val in enumerate(corr_df):\n    for col, col_val in enumerate(corr_df):\n        if row < col:\n            break\n        corr_val = corr_df.iloc[row, col]\n        if abs(corr_val) >= 0.98:\n            High_correlation.append((col_val, row_val, corr_val))\nfor entr in High_correlation:\n    print(\"{} is highly correlated with {} at a pearson correlation score of {:.3f}\". format(entr[0],entr[1], entr[2]))\ndrop_columns.extend([val[1] for val in High_correlation])\nif len(High_correlation) == 0:\n    print(\"No new columns found in risk for dropping due to high correlation\")\nelse:\n    print(\"columns to be dropped due to high correlation:\", drop_columns)\n    print(\"a total of {} columns where added\".format(len(High_correlation)))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:34:22.012276Z","iopub.execute_input":"2022-07-27T14:34:22.013487Z","iopub.status.idle":"2022-07-27T14:34:22.035905Z","shell.execute_reply.started":"2022-07-27T14:34:22.013443Z","shell.execute_reply":"2022-07-27T14:34:22.034430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the Balance variables","metadata":{}},{"cell_type":"code","source":"plot_df=df[balance_cols]\npal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']\nrow_num = int((len(balance_cols)-1)/5)+1\nfig, ax = plt.subplots(row_num,5, figsize=(16,16*(row_num/5)))\nfig.suptitle('Distribution of Risk Variables',fontsize=16)\nrow=0\ncol=[0,1,2,3,4]*row_num\nfor i, column in enumerate(plot_df.columns[:-1]):\n    if (i!=0)&(i%5==0):\n        row+=1\n    sns.kdeplot(x=column, hue='target', palette=pal[::-1], hue_order=[1,0], \n                label=['Default','Paid'], data=plot_df, \n                fill=True, linewidth=2, legend=False, ax=ax[row,col[i]])\n    ax[row,col[i]].tick_params(left=False,bottom=False)\n    ax[row,col[i]].set(title='\\n\\n{}'.format(column), xlabel='', ylabel=('Density' if i%5==0 else ''))\nfor i in range(4,5):\n    ax[row_num-1,i].set_visible(False)\nhandles, _ = ax[0,0].get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.21, 0.953))\nsns.despine(trim=True)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:34:22.037848Z","iopub.execute_input":"2022-07-27T14:34:22.038558Z","iopub.status.idle":"2022-07-27T14:35:46.417895Z","shell.execute_reply.started":"2022-07-27T14:34:22.038519Z","shell.execute_reply":"2022-07-27T14:35:46.416771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(16,16))\ncorr_df = df[balance_cols].corr()\nmask=np.triu(np.ones_like(corr_df, dtype=bool))[1:,:-1]\ncorr_df=corr_df.iloc[1:,:-1].copy()\nfig.suptitle('Pearson Correlation of Balance Variables',fontsize=16)\nsns.heatmap(corr_df, mask=mask, ax=ax, annot=True, fmt=\".2f\", annot_kws={'fontsize':8,'fontweight':'bold'}, cbar=False)# , cmap=\"YlGnBu\" , annot=True","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:35:46.419737Z","iopub.execute_input":"2022-07-27T14:35:46.420386Z","iopub.status.idle":"2022-07-27T14:35:50.672472Z","shell.execute_reply.started":"2022-07-27T14:35:46.420344Z","shell.execute_reply":"2022-07-27T14:35:50.671184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"High_correlation=[]\nfor row, row_val in enumerate(corr_df):\n    for col, col_val in enumerate(corr_df):\n        if row < col:\n            break\n        corr_val = corr_df.iloc[row, col]\n        if abs(corr_val) >= 0.98:\n            High_correlation.append((col_val, row_val, corr_val))\nfor entr in High_correlation:\n    print(\"{} is highly correlated with {} at a pearson correlation score of {:.3f}\". format(entr[0],entr[1], entr[2]))\ndrop_columns.extend([val[1] for val in High_correlation])\nif len(High_correlation) == 0:\n    print(\"No new columns found from the Balance variables for dropping due to high correlation\")\nelse:\n    print(\"columns to be dropped due to high correlation:\", drop_columns)\n    print(\"a total of {} columns where added\".format(len(High_correlation)))","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:35:50.674685Z","iopub.execute_input":"2022-07-27T14:35:50.675971Z","iopub.status.idle":"2022-07-27T14:35:50.707252Z","shell.execute_reply.started":"2022-07-27T14:35:50.675913Z","shell.execute_reply":"2022-07-27T14:35:50.705834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the payment variables","metadata":{}},{"cell_type":"code","source":"plot_df=df[payment_cols]\npal, color=['#016CC9','#DEB078'], ['#8DBAE2','#EDD3B3']\nfig, ax = plt.subplots(1,3, figsize=(9,3))\nfig.suptitle('Distribution of Payment Variables',fontsize=16)\nfor i, column in enumerate(plot_df.columns[0:-1]):\n    sns.kdeplot(x=column, hue='target', palette=pal[::-1], hue_order=[1,0], \n                label=['Default','Paid'], data=plot_df, \n                fill=True, linewidth=2, legend=False, ax=ax[i])\n    ax[i].tick_params(left=False,bottom=False)\n    ax[i].set(title='\\n\\n{}'.format(column), xlabel='', ylabel=('Density' if i%5==0 else ''))\nhandles, _ = ax[0].get_legend_handles_labels() \nfig.legend(labels=['Default','Paid'], handles=reversed(handles), ncol=2, bbox_to_anchor=(0.2, 0.995))\n#sns.despine(trim=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T14:35:50.709510Z","iopub.execute_input":"2022-07-27T14:35:50.711512Z","iopub.status.idle":"2022-07-27T14:35:58.617582Z","shell.execute_reply.started":"2022-07-27T14:35:50.711409Z","shell.execute_reply":"2022-07-27T14:35:58.615935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(3,3))\ncorr_df = df[payment_cols].corr()\nmask=np.triu(np.ones_like(corr_df, dtype=bool))[1:,:-1]\ncorr_df=corr_df.iloc[1:,:-1].copy()\nfig.suptitle('Pearson Correlation of Payment Variables',fontsize=16)\nsns.heatmap(corr_df, mask=mask, ax=ax, annot=True, fmt=\".2f\", annot_kws={'fontsize':8,'fontweight':'bold'}, cbar=False)# , cmap=\"YlGnBu\" , annot=True","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-07-27T14:35:58.619487Z","iopub.execute_input":"2022-07-27T14:35:58.619996Z","iopub.status.idle":"2022-07-27T14:35:58.836097Z","shell.execute_reply.started":"2022-07-27T14:35:58.619956Z","shell.execute_reply":"2022-07-27T14:35:58.834848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering #","metadata":{}},{"cell_type":"markdown","source":"1. Drop coloumns due to high correlation\n2. Impute values based on NaN entries\n3. One-Hot Encoding of of Categorical Variables","metadata":{}},{"cell_type":"code","source":"# Step 1\ndf.drop(axis = 1, columns=drop_columns, inplace=True,)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T14:35:58.837650Z","iopub.execute_input":"2022-07-27T14:35:58.838022Z","iopub.status.idle":"2022-07-27T14:35:59.263574Z","shell.execute_reply.started":"2022-07-27T14:35:58.837986Z","shell.execute_reply":"2022-07-27T14:35:59.262681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Step 2\n### WIP ###","metadata":{"execution":{"iopub.status.busy":"2022-07-27T14:35:59.267626Z","iopub.execute_input":"2022-07-27T14:35:59.268680Z","iopub.status.idle":"2022-07-27T14:35:59.273102Z","shell.execute_reply.started":"2022-07-27T14:35:59.268613Z","shell.execute_reply":"2022-07-27T14:35:59.271689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]}]}