{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom matplotlib import gridspec\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport missingno as msnum \n\npd.options.display.float_format = '{:.5f}'.format\npd.set_option('display.max_rows', 500)\n\n# LOAD TRAIN DATA AND MERGE TARGETS ONTO FEATURES\ndf = pd.read_csv('../input/amex-default-prediction/train_data.csv', nrows=100_000)\ndf.S_2 = pd.to_datetime(df.S_2)\ndf2 = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ndf = df.merge(df2,on='customer_ID',how='left')\ndel (df2)\ngc.collect()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-05T05:33:55.420348Z","iopub.execute_input":"2022-06-05T05:33:55.420915Z","iopub.status.idle":"2022-06-05T05:34:04.843819Z","shell.execute_reply.started":"2022-06-05T05:33:55.420826Z","shell.execute_reply":"2022-06-05T05:34:04.842935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets looks at the distribution of NA across dataset\nAlso lets remove the columns which contains 80% of NA values","metadata":{}},{"cell_type":"code","source":"col_na_count=(df.isna().sum()/df.shape[0]).reset_index(name='total')\nrem_cols=col_na_count[col_na_count['total'] > 0.70]['index'].values.tolist()\ndf.drop(rem_cols, axis=1, inplace = True)\ncol_na_count=(df.isna().sum()/df.shape[0]).reset_index(name='total')","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:04.845275Z","iopub.execute_input":"2022-06-05T05:34:04.845586Z","iopub.status.idle":"2022-06-05T05:34:05.140764Z","shell.execute_reply.started":"2022-06-05T05:34:04.845559Z","shell.execute_reply":"2022-06-05T05:34:05.140091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Analysis of missing values in the column","metadata":{}},{"cell_type":"code","source":"ax=col_na_count.plot(x='index', y='total',figsize=(40,15))\n\nax.set_xticks(range(len(col_na_count)));\nax.set_xticklabels([\"%s\" % item for item in  col_na_count['index'].tolist()], rotation=90);","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:05.142001Z","iopub.execute_input":"2022-06-05T05:34:05.142837Z","iopub.status.idle":"2022-06-05T05:34:06.742729Z","shell.execute_reply.started":"2022-06-05T05:34:05.142793Z","shell.execute_reply":"2022-06-05T05:34:06.742013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets look at the Delinquency variables ","metadata":{}},{"cell_type":"code","source":"### Helper funciton\ndef plot_graphs(x,i, type_of_graph='normal'):\n    y='target'\n    perc=(df[df['target']==1].shape[0]/df.shape[0])*100\n    \n    if type_of_graph == 'catplot':\n        df1 = df.groupby(x)[y].value_counts(normalize=True)\n        df1 = df1.mul(100)\n        df1 = df1.rename('percent').reset_index()\n        g = sns.catplot(x=x,y='percent',hue=y,kind='bar',data=df1, height=10, aspect=0.9)\n        g.ax.set_ylim(0,100)\n        plt.axhline(y=perc, color='r', linestyle='-',label=\"% of defaulters\")\n        plt.text(0,perc,\"Baseline credit default Percentage\")\n        for axes in g.axes.flat:\n            _ = axes.set_xticklabels(axes.get_xticklabels(), rotation=90)\n        for p in g.ax.patches:\n            txt = str(p.get_height().round(2)) + '%'\n            txt_x = p.get_x() \n            txt_y = p.get_height()\n            g.ax.text(txt_x,txt_y,txt)\n    else:\n        total = float(len(df))\n        plt.figure(figsize=(7, 6))\n        ax=sns.countplot(x = df[x])\n        for p in ax.patches:\n            txt = str(((p.get_height()/total)*100).round(2)) + '%'\n            txt_x = p.get_x() \n            txt_y = p.get_height()\n            ax.text(txt_x,txt_y,txt)\n        plt.xticks(rotation=90)\n        plt.show()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-06-05T05:34:06.744364Z","iopub.execute_input":"2022-06-05T05:34:06.744995Z","iopub.status.idle":"2022-06-05T05:34:06.758131Z","shell.execute_reply.started":"2022-06-05T05:34:06.744964Z","shell.execute_reply":"2022-06-05T05:34:06.757072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delinq_cols = [col for col in df.columns if 'D_' in col]","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:06.759557Z","iopub.execute_input":"2022-06-05T05:34:06.760131Z","iopub.status.idle":"2022-06-05T05:34:06.772693Z","shell.execute_reply.started":"2022-06-05T05:34:06.760098Z","shell.execute_reply":"2022-06-05T05:34:06.771797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_delinq_cols = df[delinq_cols].select_dtypes(include='number').columns.tolist()\ncategorical_delinq_cols = df[delinq_cols].select_dtypes(exclude='number').columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:06.77391Z","iopub.execute_input":"2022-06-05T05:34:06.774821Z","iopub.status.idle":"2022-06-05T05:34:06.847652Z","shell.execute_reply.started":"2022-06-05T05:34:06.77479Z","shell.execute_reply":"2022-06-05T05:34:06.846799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,cat in enumerate(categorical_delinq_cols):\n    plot_graphs(cat,i,'catplot')","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:06.84891Z","iopub.execute_input":"2022-06-05T05:34:06.849329Z","iopub.status.idle":"2022-06-05T05:34:07.815044Z","shell.execute_reply.started":"2022-06-05T05:34:06.849288Z","shell.execute_reply":"2022-06-05T05:34:07.814127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Identify relation between different variables","metadata":{}},{"cell_type":"code","source":"d = {'color': ['r', 'b']} \ng = sns.FacetGrid(df, col=\"D_63\",  row=\"target\",hue_kws=d, hue='target')\ng.map_dataframe(sns.histplot, x=\"D_64\")","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:07.818158Z","iopub.execute_input":"2022-06-05T05:34:07.818504Z","iopub.status.idle":"2022-06-05T05:34:11.842872Z","shell.execute_reply.started":"2022-06-05T05:34:07.818473Z","shell.execute_reply":"2022-06-05T05:34:11.841923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_delinq_cols=numerical_delinq_cols+['target']\n# create the figure and axes\nfig, axes = plt.subplots(9, 9,figsize=(20,20))\naxes = axes.ravel()  # flattening the array makes indexing easier\n\nfor col, ax in zip(numerical_delinq_cols, axes):\n    sns.kdeplot(data=df, x=col,hue='target', ax=ax,warn_singular=False)\n\nfig.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:11.844108Z","iopub.execute_input":"2022-06-05T05:34:11.844629Z","iopub.status.idle":"2022-06-05T05:34:57.289274Z","shell.execute_reply.started":"2022-06-05T05:34:11.8446Z","shell.execute_reply":"2022-06-05T05:34:57.288222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[numerical_delinq_cols].hist(bins=20, figsize=(14,10), color='g')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:34:57.291592Z","iopub.execute_input":"2022-06-05T05:34:57.29191Z","iopub.status.idle":"2022-06-05T05:35:05.882208Z","shell.execute_reply.started":"2022-06-05T05:34:57.291881Z","shell.execute_reply":"2022-06-05T05:35:05.881171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets look at correlation as well","metadata":{}},{"cell_type":"code","source":"correlations_target = abs(df[numerical_delinq_cols].corr())\n\n# Select upper triangle of correlation matrix\nupper = correlations_target.where(np.triu(np.ones(correlations_target.shape), k=1).astype(np.bool))\nto_drop = [column for column in upper.columns if any(upper[column] > 0.95)]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(100, 50))\ndd = [x for x in numerical_delinq_cols if x not in to_drop]\nsns.heatmap(df[dd].corr(), \n            annot=True, cmap='Spectral')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-05T05:35:05.883976Z","iopub.execute_input":"2022-06-05T05:35:05.884464Z","iopub.status.idle":"2022-06-05T05:35:06.002825Z","shell.execute_reply.started":"2022-06-05T05:35:05.884422Z","shell.execute_reply":"2022-06-05T05:35:06.001899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We have reduced our original Delinquecy columns from 76 to 69 just based on correlation. We should be able to look at the target distribution and shortlist a set of columns that will be useful for our predictions","metadata":{}},{"cell_type":"markdown","source":"### WIP","metadata":{"execution":{"iopub.status.busy":"2022-06-04T11:43:53.892541Z","iopub.execute_input":"2022-06-04T11:43:53.89292Z","iopub.status.idle":"2022-06-04T11:43:53.898774Z","shell.execute_reply.started":"2022-06-04T11:43:53.89289Z","shell.execute_reply":"2022-06-04T11:43:53.89816Z"}}}]}