{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AMEX Competition - Data Exploration\n\nThis notebook was created during a live coding session on twitch.\n\nCheck out the VOD video of this stream and follow for future streams [here](https://www.twitch.tv/medallionstallion_)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('seaborn')\ncolor_pal = sns.color_palette()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-27T01:59:26.473001Z","iopub.execute_input":"2022-05-27T01:59:26.473537Z","iopub.status.idle":"2022-05-27T01:59:27.595796Z","shell.execute_reply.started":"2022-05-27T01:59:26.473499Z","shell.execute_reply":"2022-05-27T01:59:27.594417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading in the Dataset\nWe will use the parquet format of the dataset created by @odins0n for some data exploration. Parquet format is faster, more compressed, and saves the dtypes of each column when we read and write.\n\n[Learn more about it here in the youtube video I made.](https://www.youtube.com/watch?v=u4rsA5ZiTls)\n\nWe will subsample the training data so that the notebook does not run out of memory.","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('../input/amex-parquet/train_data.parquet')\nprint(f'The full training data shape is: {train.shape}')\ntrain = train.sample(100_000, random_state=529)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T01:26:27.414736Z","iopub.execute_input":"2022-05-27T01:26:27.415222Z","iopub.status.idle":"2022-05-27T01:27:03.827163Z","shell.execute_reply.started":"2022-05-27T01:26:27.415185Z","shell.execute_reply":"2022-05-27T01:27:03.821806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# About the Data:\nThe features are broken down into types. We will explore each one:\n\n- D_* = Delinquency variables\n- S_* = Spend variables\n- P_* = Payment variables\n- B_* = Balance variables\n- R_* = Risk variables","metadata":{}},{"cell_type":"code","source":"d_feats = [c for c in train.columns if c.startswith('D_')]\ns_feats = [c for c in train.columns if c.startswith('S_')]\np_feats = [c for c in train.columns if c.startswith('P_')]\nb_feats = [c for c in train.columns if c.startswith('B_')]\nr_feats = [c for c in train.columns if c.startswith('R_')]","metadata":{"execution":{"iopub.status.busy":"2022-05-27T01:43:59.883223Z","iopub.execute_input":"2022-05-27T01:43:59.885241Z","iopub.status.idle":"2022-05-27T01:43:59.896534Z","shell.execute_reply.started":"2022-05-27T01:43:59.885145Z","shell.execute_reply":"2022-05-27T01:43:59.894813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of Delinquency variables: {len(d_feats)}')\nprint(f'Number of Spend variables: {len(s_feats)}')\nprint(f'Number of Payment variables: {len(p_feats)}')\nprint(f'Number of Balance variables: {len(b_feats)}')\nprint(f'Number of Risk variables: {len(r_feats)}')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T01:47:48.669567Z","iopub.execute_input":"2022-05-27T01:47:48.670228Z","iopub.status.idle":"2022-05-27T01:47:48.679575Z","shell.execute_reply.started":"2022-05-27T01:47:48.670112Z","shell.execute_reply":"2022-05-27T01:47:48.678505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of the Target","metadata":{}},{"cell_type":"code","source":"pct_default = train['target'].mean()\nprint(f'{(pct_default *100): 0.2f}% of the Training Data Defaults')\ntrain['target'].value_counts() \\\n    .plot(kind='barh',\n          title='Distribution of Target',\n          color=color_pal[1])","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:27:31.164905Z","iopub.execute_input":"2022-05-27T02:27:31.165327Z","iopub.status.idle":"2022-05-27T02:27:31.358410Z","shell.execute_reply.started":"2022-05-27T02:27:31.165297Z","shell.execute_reply":"2022-05-27T02:27:31.357093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How many Null Values by Features","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 1, figsize=(10, 20))\ntrain[d_feats].isna().mean() \\\n    .plot(kind='hist', bins=20, color=color_pal[0], ax=axs[0])\naxs[0].set_title('Null Values in Delinquency variables', fontsize=20)\naxs[0].set_xlabel('Percent of Null Values')\n\ntrain[s_feats].isna().mean() \\\n    .plot(kind='hist', bins=20, color=color_pal[1], ax=axs[1])\naxs[1].set_title('Null Values in Spend variables', fontsize=20)\naxs[1].set_xlabel('Percent of Null Values')\n\ntrain[p_feats].isna().mean() \\\n    .plot(kind='hist', bins=20, color=color_pal[2], ax=axs[2])\naxs[2].set_title('Null Values in Payment variables', fontsize=20)\naxs[2].set_xlabel('Percent of Null Values')\n\ntrain[b_feats].isna().mean() \\\n    .plot(kind='hist', bins=20, color=color_pal[3], ax=axs[3])\naxs[3].set_title('Null Values in Balance variables', fontsize=20)\naxs[3].set_xlabel('Percent of Null Values')\n\ntrain[r_feats].isna().mean() \\\n    .plot(kind='hist', bins=20, color=color_pal[4], ax=axs[4])\naxs[4].set_title('Null Values in Risk variables', fontsize=20)\naxs[4].set_xlabel('Percent of Null Values')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:07:36.876723Z","iopub.execute_input":"2022-05-27T02:07:36.877259Z","iopub.status.idle":"2022-05-27T02:07:38.011542Z","shell.execute_reply.started":"2022-05-27T02:07:36.877221Z","shell.execute_reply":"2022-05-27T02:07:38.010315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at 10 D features\ntrain[d_feats[:10]].describe().T","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:13:43.349307Z","iopub.execute_input":"2022-05-27T02:13:43.349718Z","iopub.status.idle":"2022-05-27T02:13:43.444992Z","shell.execute_reply.started":"2022-05-27T02:13:43.349683Z","shell.execute_reply":"2022-05-27T02:13:43.443614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = train[d_feats[:10]] \\\n    .plot(kind='kde', figsize=(10, 5))\nax.set_title('Distribution of 10 D_ features')\nax.set_xlim(-0.5, 1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:18:38.237634Z","iopub.execute_input":"2022-05-27T02:18:38.239428Z","iopub.status.idle":"2022-05-27T02:18:52.290501Z","shell.execute_reply.started":"2022-05-27T02:18:38.239352Z","shell.execute_reply":"2022-05-27T02:18:52.289486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = train[p_feats] \\\n    .plot(kind='kde', figsize=(10, 5))\nax.set_title('Distribution of Payment features')\nax.set_xlim(-0.5, 1.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:20:57.236254Z","iopub.execute_input":"2022-05-27T02:20:57.236764Z","iopub.status.idle":"2022-05-27T02:21:02.831401Z","shell.execute_reply.started":"2022-05-27T02:20:57.236727Z","shell.execute_reply":"2022-05-27T02:21:02.830102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Each Feature by Target","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 3, figsize=(10, 3))\nfor i in range(3):\n    train.groupby('target')[p_feats[i]] \\\n        .plot(kind='kde',\n              title=p_feats[i], alpha=0.5, ax=axs[i])\n    axs[i].legend()\nfig.suptitle('Distribution Payment Features by Target',\n             y=1.05, fontsize=14)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:33:39.639763Z","iopub.execute_input":"2022-05-27T02:33:39.640574Z","iopub.status.idle":"2022-05-27T02:33:45.719427Z","shell.execute_reply.started":"2022-05-27T02:33:39.640505Z","shell.execute_reply":"2022-05-27T02:33:45.718284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Non-Numeric Features\n\nThere are some features that are non-numeric!\n\nThey are:\n- S_2\n- D_63\n- D_64","metadata":{}},{"cell_type":"code","source":"train.select_dtypes('object').columns","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:42:39.238342Z","iopub.execute_input":"2022-05-27T02:42:39.238821Z","iopub.status.idle":"2022-05-27T02:42:39.261844Z","shell.execute_reply.started":"2022-05-27T02:42:39.238782Z","shell.execute_reply":"2022-05-27T02:42:39.260341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['S_2_date'] = pd.to_datetime(train['S_2'])","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:42:57.566458Z","iopub.execute_input":"2022-05-27T02:42:57.566972Z","iopub.status.idle":"2022-05-27T02:42:57.588678Z","shell.execute_reply.started":"2022-05-27T02:42:57.566932Z","shell.execute_reply":"2022-05-27T02:42:57.587083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## S_2 (Date) Feature Exploration","metadata":{}},{"cell_type":"code","source":"train.set_index('S_2_date')['target'] \\\n    .plot(figsize=(15, 5), lw=1, alpha=0.5,\n          title='Target by Date')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:44:23.213901Z","iopub.execute_input":"2022-05-27T02:44:23.214472Z","iopub.status.idle":"2022-05-27T02:44:23.973798Z","shell.execute_reply.started":"2022-05-27T02:44:23.214432Z","shell.execute_reply":"2022-05-27T02:44:23.972073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categorical Feature Exploration","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:52:33.307575Z","iopub.execute_input":"2022-05-27T02:52:33.308118Z","iopub.status.idle":"2022-05-27T02:52:33.337206Z","shell.execute_reply.started":"2022-05-27T02:52:33.308081Z","shell.execute_reply":"2022-05-27T02:52:33.336129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('D_63')['target'].value_counts() \\\n    .unstack() \\\n    .sort_values(0) \\\n    .plot(kind='barh', stacked=True,\n                    title='D_63 Feature by Target')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:52:46.073082Z","iopub.execute_input":"2022-05-27T02:52:46.073541Z","iopub.status.idle":"2022-05-27T02:52:46.347832Z","shell.execute_reply.started":"2022-05-27T02:52:46.073504Z","shell.execute_reply":"2022-05-27T02:52:46.346732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('D_64')['target'].value_counts() \\\n    .unstack() \\\n    .sort_values(0) \\\n    .plot(kind='barh', stacked=True,\n                    title='D_64 Feature by Target')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:52:57.635494Z","iopub.execute_input":"2022-05-27T02:52:57.635902Z","iopub.status.idle":"2022-05-27T02:52:59.857442Z","shell.execute_reply.started":"2022-05-27T02:52:57.635871Z","shell.execute_reply":"2022-05-27T02:52:59.856648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find the Correlation of Features with Target","metadata":{}},{"cell_type":"code","source":"numeric_feats = train.select_dtypes('float32').columns\nfeats = [c for c in train.columns if c not in ['customer_ID', 'target']]\nfeat_corrs = {}\nfor f in numeric_feats:\n    feat_corr = np.corrcoef(train[f].fillna(0), train['target'])[0, 1]\n    feat_corrs[f] = feat_corr","metadata":{"execution":{"iopub.status.busy":"2022-05-27T02:58:38.256568Z","iopub.execute_input":"2022-05-27T02:58:38.257850Z","iopub.status.idle":"2022-05-27T02:58:38.553788Z","shell.execute_reply.started":"2022-05-27T02:58:38.257791Z","shell.execute_reply":"2022-05-27T02:58:38.552467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most and Least Correlated Features with the Target","metadata":{}},{"cell_type":"code","source":"pd.Series(feat_corrs).abs().sort_values(ascending=False).head(25) \\\n     .sort_values() \\\n    .plot(kind='barh', title='Top Correlated Features with Target')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:02:27.590223Z","iopub.execute_input":"2022-05-27T03:02:27.591975Z","iopub.status.idle":"2022-05-27T03:02:27.920728Z","shell.execute_reply.started":"2022-05-27T03:02:27.591898Z","shell.execute_reply":"2022-05-27T03:02:27.919065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.Series(feat_corrs).abs().sort_values(ascending=True).head(25) \\\n    .plot(kind='barh', title='Least Correlated Features with Target')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:02:51.602891Z","iopub.execute_input":"2022-05-27T03:02:51.603409Z","iopub.status.idle":"2022-05-27T03:02:51.909281Z","shell.execute_reply.started":"2022-05-27T03:02:51.603373Z","shell.execute_reply":"2022-05-27T03:02:51.908178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = train.groupby('target')['D_48'] \\\n    .plot(kind='kde',\n          title='D_48', alpha=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:04:03.857591Z","iopub.execute_input":"2022-05-27T03:04:03.858125Z","iopub.status.idle":"2022-05-27T03:04:05.726742Z","shell.execute_reply.started":"2022-05-27T03:04:03.858083Z","shell.execute_reply":"2022-05-27T03:04:05.725864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}