{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**In this notebook, I studied the following relationships:**\n\n*Association between categorical features (feature-feature)*\n\n*Association between categorical features and target (feature-target)*\n\n*Correlation between numerical features (feature-feature)*\n\n*Correlation between numerical features and target (feature-target)*\n\n**I highly recommend reading this medium post which explains my approach in this notebook.**\n\nThe Search for Categorical Correlation: https://towardsdatascience.com/the-search-for-categorical-correlation-a1cf7f1888c9\n\n\n**To calculate the association and correlations, I used Dython which is a set of data analysis tools in PYTHON 3.x, which can let you get more insights into your data.**\n\ndython website: http://shakedzy.xyz/dython/\n\n\n**The output of this notebook:**\n\n*cat_f_f.csv* Association between categorical features (feature-feature)\n\n*cat_f_t.csv* Association between categorical features and target (feature-target)\n\n*num_f_f.csv* Correlation between numerical features (feature-feature)\n\n*num_f_t.csv* Correlation between numerical features and target (feature-target)\n\n\n**Feel free to download and use the output of this notebook for your feature engineering studies.**\n\nGood Luck!","metadata":{}},{"cell_type":"code","source":"# The easiest way to install dython is using pip install:\n!pip install dython","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:06:45.146976Z","iopub.execute_input":"2022-06-01T11:06:45.147442Z","iopub.status.idle":"2022-06-01T11:06:58.200498Z","shell.execute_reply.started":"2022-06-01T11:06:45.147351Z","shell.execute_reply":"2022-06-01T11:06:58.199232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom dython.nominal import associations\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:06:58.202899Z","iopub.execute_input":"2022-06-01T11:06:58.203293Z","iopub.status.idle":"2022-06-01T11:06:59.947789Z","shell.execute_reply.started":"2022-06-01T11:06:58.203257Z","shell.execute_reply":"2022-06-01T11:06:59.946905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\ntrain_data_org = pd.read_feather('../input/amex-default-prediction-feather/train.feather').set_index('customer_ID')\nif DEBUG:\n    train_data = train_data_org.iloc[:20000, :].copy()\nelse:\n    train_data = train_data_org.copy()  \ndel train_data_org\ntrain_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv', index_col='customer_ID').loc[train_data.index]\ncategorical_features = ['B_30', 'B_31', 'B_38', 'D_114', \n                        'D_116', 'D_117', 'D_120', 'D_126', \n                        'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:06:59.949363Z","iopub.execute_input":"2022-06-01T11:06:59.949810Z","iopub.status.idle":"2022-06-01T11:07:26.935974Z","shell.execute_reply.started":"2022-06-01T11:06:59.949768Z","shell.execute_reply":"2022-06-01T11:07:26.934113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape, train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:07:26.939131Z","iopub.execute_input":"2022-06-01T11:07:26.939499Z","iopub.status.idle":"2022-06-01T11:07:26.950668Z","shell.execute_reply.started":"2022-06-01T11:07:26.939464Z","shell.execute_reply":"2022-06-01T11:07:26.949632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"association_dictionary = associations(train_data[categorical_features], nominal_columns = categorical_features, mark_columns = True, \n                nom_nom_assoc = 'theil', nan_strategy = 'drop_samples', figsize= (15, 15), vmin = 0, vmax=0.8, compute_only = False)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:07:31.928477Z","iopub.execute_input":"2022-06-01T11:07:31.929162Z","iopub.status.idle":"2022-06-01T11:07:38.127997Z","shell.execute_reply.started":"2022-06-01T11:07:31.929114Z","shell.execute_reply":"2022-06-01T11:07:38.126956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('associations ranking:')\nacff = association_dictionary['corr'].stack()\nacff = acff[acff.index.get_level_values(0) < acff.index.get_level_values(1)]\nacff = acff.sort_values(ascending = False)\nacff.to_csv('cat_f_f.csv')\nprint(acff)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:08:01.571252Z","iopub.execute_input":"2022-06-01T11:08:01.572070Z","iopub.status.idle":"2022-06-01T11:08:01.596842Z","shell.execute_reply.started":"2022-06-01T11:08:01.572021Z","shell.execute_reply":"2022-06-01T11:08:01.595678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_target = pd.concat([train_data.groupby('customer_ID').tail(1), train_labels.groupby('customer_ID').tail(1)], axis=1)\ntrain_feature_target.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:08:33.190741Z","iopub.execute_input":"2022-06-01T11:08:33.191964Z","iopub.status.idle":"2022-06-01T11:08:33.244769Z","shell.execute_reply.started":"2022-06-01T11:08:33.191870Z","shell.execute_reply":"2022-06-01T11:08:33.243992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nThe association between a categorical target and categorical features:')\nassociation_dictionary = associations(train_feature_target[categorical_features + ['target']], \n                                      nominal_columns = categorical_features + ['target'], mark_columns = True,\n                                      display_rows = ['target'], nan_strategy = 'drop_samples',figsize= (15, 15),\n                                      vmin = 0, vmax=0.8, compute_only = False, cbar = False)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:08:41.980944Z","iopub.execute_input":"2022-06-01T11:08:41.981359Z","iopub.status.idle":"2022-06-01T11:08:43.121549Z","shell.execute_reply.started":"2022-06-01T11:08:41.981327Z","shell.execute_reply":"2022-06-01T11:08:43.120456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acft = association_dictionary['corr'].stack().sort_values(ascending = False)\nprint(acft)\nacft.to_csv('cat_f_t.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:08:57.938172Z","iopub.execute_input":"2022-06-01T11:08:57.938628Z","iopub.status.idle":"2022-06-01T11:08:57.952474Z","shell.execute_reply.started":"2022-06-01T11:08:57.938586Z","shell.execute_reply":"2022-06-01T11:08:57.951399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_features = list(set(train_data.columns).difference(set(categorical_features)))\nnumerical_features.remove('S_2')\ntrain_feature_target[numerical_features] = train_feature_target[numerical_features].fillna(train_feature_target[numerical_features].median())","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:09:29.069314Z","iopub.execute_input":"2022-06-01T11:09:29.070293Z","iopub.status.idle":"2022-06-01T11:09:29.148871Z","shell.execute_reply.started":"2022-06-01T11:09:29.070252Z","shell.execute_reply":"2022-06-01T11:09:29.147904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compute_only = True means that we don't want to plot the heatmap \ncorrelation_dictionary = associations(train_feature_target[numerical_features], numerical_columns = numerical_features, mark_columns = False, \n                num_num_assoc = 'pearson', nan_strategy = 'drop_samples',  figsize= (15, 15), vmin = 0, vmax=0.8, compute_only = True)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:10:25.459260Z","iopub.execute_input":"2022-06-01T11:10:25.460207Z","iopub.status.idle":"2022-06-01T11:10:35.083306Z","shell.execute_reply.started":"2022-06-01T11:10:25.460157Z","shell.execute_reply":"2022-06-01T11:10:35.082063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('correlations ranking:')\ncnff = correlation_dictionary['corr'].stack()\ncnff = cnff[cnff.index.get_level_values(0) < cnff.index.get_level_values(1)]\ncnff = cnff.sort_values(ascending = False)\ncnff.to_csv('num_f_f.csv')\nprint(cnff)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:11:34.056280Z","iopub.execute_input":"2022-06-01T11:11:34.056665Z","iopub.status.idle":"2022-06-01T11:11:34.131360Z","shell.execute_reply.started":"2022-06-01T11:11:34.056635Z","shell.execute_reply":"2022-06-01T11:11:34.130416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlation_dictionary = associations(train_feature_target[numerical_features + ['target']], nominal_columns = 'target',\n                                        numerical_columns = numerical_features, mark_columns = True, \n                                        display_rows = ['target'], nan_strategy = 'drop_samples', \n                                        figsize= (15, 15), vmin = 0, vmax=0.8, compute_only = True, cbar = False)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:12:55.918790Z","iopub.execute_input":"2022-06-01T11:12:55.919380Z","iopub.status.idle":"2022-06-01T11:13:05.881806Z","shell.execute_reply.started":"2022-06-01T11:12:55.919338Z","shell.execute_reply":"2022-06-01T11:13:05.880812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnft = correlation_dictionary['corr'].stack().sort_values(ascending = False)\ncnft.to_csv('num_f_t.csv')\nprint(cnft)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T11:13:16.830052Z","iopub.execute_input":"2022-06-01T11:13:16.830487Z","iopub.status.idle":"2022-06-01T11:13:16.843577Z","shell.execute_reply.started":"2022-06-01T11:13:16.830453Z","shell.execute_reply":"2022-06-01T11:13:16.842320Z"},"trusted":true},"execution_count":null,"outputs":[]}]}