{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# American Express EDA\n\n## Libraries & Utilities","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\n\npalette = 'Pastel1'\n%matplotlib inline\nsns.set_style('darkgrid')\nsns.set_context(rc={\"grid.linewidth\": 3.2})\ncolors = ['#DF6565', '#BDDB67', '#67BBDB', '#AF67DB', '#D9DB67']","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-29T06:07:49.487987Z","iopub.execute_input":"2022-07-29T06:07:49.488444Z","iopub.status.idle":"2022-07-29T06:07:50.902256Z","shell.execute_reply.started":"2022-07-29T06:07:49.488341Z","shell.execute_reply":"2022-07-29T06:07:50.901324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n## 📂 Files\n\n- **train_data.csv** - training data with multiple statement dates per customer_ID\n- **train_labels.csv** - target label for each customer_ID\n- **test_data.csv** - corresponding test data; your objective is to predict the target label for each customer_ID\n- **sample_submission.csv** - a sample submission file in the correct format\n\n## Loading Data","metadata":{}},{"cell_type":"code","source":"ftr_path =  '/kaggle/input/parquet-files-amexdefault-prediction/'\ncsv_path = '/kaggle/input/amex-default-prediction/'\n\ntrain = pd.read_feather(os.path.join(ftr_path, 'train_data.ftr'))\n\ntest = pd.read_feather(os.path.join(ftr_path, 'test_data.ftr'))\n\ntrain_labels = pd.read_csv(os.path.join(csv_path, 'train_labels.csv'))\n\nprint(f'Train Shape: {train.shape}')\nprint(f'Test Shape: {test.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:07:50.904321Z","iopub.execute_input":"2022-07-29T06:07:50.905913Z","iopub.status.idle":"2022-07-29T06:09:11.031138Z","shell.execute_reply.started":"2022-07-29T06:07:50.905816Z","shell.execute_reply":"2022-07-29T06:09:11.029668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 📃 Variable Description\n\nThe objective of this competition is to predict the probability that a customer does not pay back their credit card balance amount in the future based on their monthly customer profile. The target binary variable is calculated by observing 18 months performance window after the latest credit card statement, and if the customer does not pay due amount in 120 days after their latest statement date it is considered a default event.\n\nThe dataset contains aggregated profile features for each customer at each statement date. Features are anonymized and normalized, and fall into the following general categories:\n\n<code>D_*</code> : **Delinquency** variables<br>\n<code>S_*</code> : **Spend** variables<br>\n<code>P_*</code> : **Payment** variables<br>\n<code>B_*</code> : **Balance** variables<br>\n<code>R_*</code> : **Risk** variables<br>\n\nwith the following features being categorical:\n\n<code>['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']</code>\n\n---","metadata":{}},{"cell_type":"code","source":"#Defining Categorical Columns\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:09:11.032785Z","iopub.execute_input":"2022-07-29T06:09:11.033868Z","iopub.status.idle":"2022-07-29T06:09:11.039521Z","shell.execute_reply.started":"2022-07-29T06:09:11.033831Z","shell.execute_reply":"2022-07-29T06:09:11.037985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"profile_freq = pd.Series([col.split('_')[0] for col in train.columns[1:]]).replace({'D': 'Delinquency',\n                                                                                    'S': 'Spend',\n                                                                                    'P': 'Payment',\n                                                                                    'B': 'Balance',\n                                                                                    'R': 'Risk'}).value_counts()\nfig = go.Figure(data=[go.Pie(labels=profile_freq.keys(),\n                             values=profile_freq.values)])\n\nfig.update_traces(hoverinfo='value',\n                  textinfo='label+value',\n                  textfont_size=16,\n                  textposition ='auto',\n                  showlegend = False,\n                  marker=dict(colors = colors)\n                 )\n\nfig.update_layout(title={'text': \"Profile Groups\",\n                         'y':0.9,\n                         'x':0.5,\n                         'xanchor': 'center',\n                         'yanchor': 'top'},\n                 template='simple_white')\niplot(fig)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:09:11.043148Z","iopub.execute_input":"2022-07-29T06:09:11.043642Z","iopub.status.idle":"2022-07-29T06:09:12.224700Z","shell.execute_reply.started":"2022-07-29T06:09:11.043594Z","shell.execute_reply":"2022-07-29T06:09:12.223125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical Analysis","metadata":{}},{"cell_type":"code","source":"for col in cat_cols:\n    fig, axes = plt.subplots(1, 2, figsize=(16, 6))\n    fig.suptitle(col, size = 14)\n    sns.countplot(ax = axes[0], data = train,\n                  x = col, order = train[col].value_counts().index, palette= palette)\n    axes[0].set_title('Train', size = 12)\n    sns.countplot(ax = axes[1], data = test,\n                  x = col, order = test[col].value_counts().index, palette= palette)\n    axes[1].set_title('Test', size = 12)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:09:12.226614Z","iopub.execute_input":"2022-07-29T06:09:12.227156Z","iopub.status.idle":"2022-07-29T06:09:23.202305Z","shell.execute_reply.started":"2022-07-29T06:09:12.227100Z","shell.execute_reply":"2022-07-29T06:09:23.200669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_cols:\n    train_but_not_test = [c for c in train[col].unique() if c not in test[col].unique()]\n    test_but_not_train = [c for c in test[col].unique() if c not in train[col].unique()]\n    if len(train_but_not_test) != 0:\n        print(f'{train_but_not_test} values are in train data but not in test data for {col}')\n    if len(test_but_not_train) != 0:\n        print(f'{test_but_not_train} values are in test data but not in train data for {col} \\n')","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:09:23.204309Z","iopub.execute_input":"2022-07-29T06:09:23.204852Z","iopub.status.idle":"2022-07-29T06:09:30.506354Z","shell.execute_reply.started":"2022-07-29T06:09:23.204804Z","shell.execute_reply":"2022-07-29T06:09:30.504682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation Coefficients","metadata":{}},{"cell_type":"code","source":"def corr_map(dataframe, method = 'pearson', title = None):\n    assert method in ['pearson', 'spearman'], 'Invalid Correlation Method'\n    sns.set_style(\"white\")\n    matrix = np.triu(dataframe.corr(method=method))\n    f,ax=plt.subplots(figsize = (matrix.shape[0]*0.75,\n                                 matrix.shape[1]*0.75))\n    sns.heatmap(dataframe.corr(method=method),\n                annot= True,\n                fmt = \".2f\",\n                cbar = False,\n                ax=ax,\n                vmin = -1,\n                vmax = 1,\n                mask = matrix,\n                cmap = \"coolwarm\",\n                linewidth = 0.4,\n                linecolor = \"white\",\n                annot_kws={\"size\": 12})\n    plt.xticks(rotation=80,size=14)\n    plt.yticks(rotation=0,size=14)\n    if title == None:\n        title = f'{method.title()} Correlation Map'\n    plt.title(title, size = 14)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:09:30.508510Z","iopub.execute_input":"2022-07-29T06:09:30.508990Z","iopub.status.idle":"2022-07-29T06:09:30.523439Z","shell.execute_reply.started":"2022-07-29T06:09:30.508938Z","shell.execute_reply":"2022-07-29T06:09:30.521465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"profile_groups = ['D', 'S', 'P', 'B', 'R']\ntrain_num_data = train.merge(train_labels, on = 'customer_ID', how = 'left')\ntrain_num_data = train_num_data[[col for col in train_num_data.columns \n                                 if col not in cat_cols and col != 'customer_ID']]\n\nfor group in profile_groups:\n    sub_cols = [col for col in train_num_data.columns if col.startswith(group)]\n    sub_cols.extend(['target'])\n    corr_map(dataframe = train_num_data[sub_cols], method = 'pearson', title = f'Profile Group: {group}')\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T06:09:30.525107Z","iopub.execute_input":"2022-07-29T06:09:30.525964Z","iopub.status.idle":"2022-07-29T06:16:37.415870Z","shell.execute_reply.started":"2022-07-29T06:09:30.525919Z","shell.execute_reply":"2022-07-29T06:16:37.414458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}