{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Amex Features EDA\n\nI've aggregated the features in the following way (please access the data here https://www.kaggle.com/datasets/viktorcikojevic/amex-preprocessed-data.\n\nThere are several aggregations in this data, aggregated over dates (the S_2 field): \"growth\" is the rate or change for each variable, \"avg\" is the average, \"stddev\" is standard deviation, \"min\" is the min for, \"max\" is maximum value and finally \"last_value\" is the value of the field on a last date in a dataset. The whole training dataset is split into 8 files.\n\nThis notebook serves as an EDA of these features: I plot the histograms of each variable, for each target. On the left (right) is the logarithmic (linear) scale.","metadata":{}},{"cell_type":"markdown","source":"# Load Train Data","metadata":{}},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport pandas as pd, numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import gridspec\n\n# LOAD TRAIN DATA AND MERGE TARGETS ONTO FEATURES\ndf = pd.read_csv('../input/amex-preprocessed-data/processed_8_fortraining000000000000.csv', nrows=100_000)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T15:31:03.154535Z","iopub.execute_input":"2022-08-07T15:31:03.155333Z","iopub.status.idle":"2022-08-07T15:31:31.029307Z","shell.execute_reply.started":"2022-08-07T15:31:03.155235Z","shell.execute_reply":"2022-08-07T15:31:31.028204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_0 = df[df[\"target\"] == 0]\ndf_1 = df[df[\"target\"] == 1]\n\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-08-07T15:31:31.031630Z","iopub.execute_input":"2022-08-07T15:31:31.032174Z","iopub.status.idle":"2022-08-07T15:31:31.421836Z","shell.execute_reply.started":"2022-08-07T15:31:31.032096Z","shell.execute_reply":"2022-08-07T15:31:31.420594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example plot\nx_0 = df_0[\"P_2_growth\"].to_numpy()\nx_1 = df_1[\"P_2_growth\"].to_numpy()\nx_0 = x_0[~np.isnan(x_0)]\nx_1 = x_1[~np.isnan(x_1)]\nx = np.append(x_0, x_1)\nplt.hist(x_0, label=\"0\", bins=100, color='orange', alpha=0.8, stacked=True)\nplt.hist(x_1, label=\"0\", bins=100, color='blue', alpha=0.8, stacked=True)\nx_ave = np.average(x)\nx_stddev = np.std(x)\nplt.xlim(x_ave - 5*x_stddev, x_ave + 5*x_stddev)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T15:31:31.423390Z","iopub.execute_input":"2022-08-07T15:31:31.423777Z","iopub.status.idle":"2022-08-07T15:31:32.016054Z","shell.execute_reply.started":"2022-08-07T15:31:31.423744Z","shell.execute_reply":"2022-08-07T15:31:32.015252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregations = [\"growth\",\n\"avg\",\n\"stddev\",\n\"min\",\n\"max\",\n\"last_value\"]\naggregations","metadata":{"execution":{"iopub.status.busy":"2022-08-07T15:31:32.017186Z","iopub.execute_input":"2022-08-07T15:31:32.018224Z","iopub.status.idle":"2022-08-07T15:31:32.025137Z","shell.execute_reply.started":"2022-08-07T15:31:32.018171Z","shell.execute_reply":"2022-08-07T15:31:32.024410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ndef plot_histograms(prefix='D', agg='avg'):\n    columns = df_0.columns.to_list()\n    for column in columns:\n        if column.replace(f\"_{agg}\", \"\") in cat_features or prefix not in column or agg not in column:\n            continue\n        x_0 = df_0[column].to_numpy()\n        x_1 = df_1[column].to_numpy()\n        x_0 = x_0[~np.isnan(x_0)]\n        x_1 = x_1[~np.isnan(x_1)]\n        x = np.append(x_0, x_1)\n        x_ave = np.average(x)\n        x_stddev = np.std(x)\n        # plt.xlim(x_ave - 5*x_stddev, x_ave + 5*x_stddev)\n        fig, axs = plt.subplots(1, 2, figsize=(20,5))\n        axs[0].hist([x_0, x_1], label=\"0\", color=['orange', 'blue'], bins=100, alpha=0.5, stacked=True)\n        axs[1].hist([x_0, x_1], label=\"0\", color=['orange', 'blue'], bins=100, alpha=0.5, stacked=True)\n        axs[0].set_title(f\"{column}, 0=orange, 1=blue\")\n        axs[1].set_title(f\"{column}, 0=orange, 1=blue\")\n        axs[0].set_yscale('log')\n        axs[1].set_yscale('linear')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T15:31:32.026713Z","iopub.execute_input":"2022-08-07T15:31:32.027836Z","iopub.status.idle":"2022-08-07T15:31:32.040306Z","shell.execute_reply.started":"2022-08-07T15:31:32.027796Z","shell.execute_reply":"2022-08-07T15:31:32.038875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Delinquency Variables","metadata":{}},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('D', agg='growth')","metadata":{"execution":{"iopub.status.busy":"2022-08-07T15:31:32.041802Z","iopub.execute_input":"2022-08-07T15:31:32.042691Z","iopub.status.idle":"2022-08-07T15:32:55.836184Z","shell.execute_reply.started":"2022-08-07T15:31:32.042655Z","shell.execute_reply":"2022-08-07T15:32:55.834550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('D', agg='avg')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:12:36.983514Z","iopub.execute_input":"2022-08-04T17:12:36.983878Z","iopub.status.idle":"2022-08-04T17:13:57.683639Z","shell.execute_reply.started":"2022-08-04T17:12:36.983847Z","shell.execute_reply":"2022-08-04T17:13:57.682667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('D', agg='stddev')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:13:57.684992Z","iopub.execute_input":"2022-08-04T17:13:57.685357Z","iopub.status.idle":"2022-08-04T17:15:19.082746Z","shell.execute_reply.started":"2022-08-04T17:13:57.685324Z","shell.execute_reply":"2022-08-04T17:15:19.081703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('D', agg='min')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:15:19.084481Z","iopub.execute_input":"2022-08-04T17:15:19.084823Z","iopub.status.idle":"2022-08-04T17:16:38.088418Z","shell.execute_reply.started":"2022-08-04T17:15:19.084792Z","shell.execute_reply":"2022-08-04T17:16:38.087430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('D', agg='max')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:38.089685Z","iopub.execute_input":"2022-08-04T17:16:38.090048Z","iopub.status.idle":"2022-08-04T17:16:51.817874Z","shell.execute_reply.started":"2022-08-04T17:16:38.090014Z","shell.execute_reply":"2022-08-04T17:16:51.813422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('D', agg='last_value')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.819694Z","iopub.status.idle":"2022-08-04T17:16:51.820481Z","shell.execute_reply.started":"2022-08-04T17:16:51.820263Z","shell.execute_reply":"2022-08-04T17:16:51.820288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Spend Variables","metadata":{}},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('S', agg='growth')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.821448Z","iopub.status.idle":"2022-08-04T17:16:51.822115Z","shell.execute_reply.started":"2022-08-04T17:16:51.821760Z","shell.execute_reply":"2022-08-04T17:16:51.821797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('S', agg='avg')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.824615Z","iopub.status.idle":"2022-08-04T17:16:51.825061Z","shell.execute_reply.started":"2022-08-04T17:16:51.824844Z","shell.execute_reply":"2022-08-04T17:16:51.824862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('S', agg='stddev')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.826022Z","iopub.status.idle":"2022-08-04T17:16:51.826393Z","shell.execute_reply.started":"2022-08-04T17:16:51.826225Z","shell.execute_reply":"2022-08-04T17:16:51.826243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('S', agg='min')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.828036Z","iopub.status.idle":"2022-08-04T17:16:51.828949Z","shell.execute_reply.started":"2022-08-04T17:16:51.828726Z","shell.execute_reply":"2022-08-04T17:16:51.828754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('S', agg='max')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.830083Z","iopub.status.idle":"2022-08-04T17:16:51.830565Z","shell.execute_reply.started":"2022-08-04T17:16:51.830321Z","shell.execute_reply":"2022-08-04T17:16:51.830350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LEAVE LIST BLANK TO PLOT ALL\nplot_histograms('S', agg='last_value')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.832319Z","iopub.status.idle":"2022-08-04T17:16:51.832882Z","shell.execute_reply.started":"2022-08-04T17:16:51.832574Z","shell.execute_reply":"2022-08-04T17:16:51.832603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Payment Variables","metadata":{}},{"cell_type":"code","source":"plot_histograms('P', agg='growth')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.835187Z","iopub.status.idle":"2022-08-04T17:16:51.835986Z","shell.execute_reply.started":"2022-08-04T17:16:51.835642Z","shell.execute_reply":"2022-08-04T17:16:51.835674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('P', agg='avg')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.837512Z","iopub.status.idle":"2022-08-04T17:16:51.838083Z","shell.execute_reply.started":"2022-08-04T17:16:51.837829Z","shell.execute_reply":"2022-08-04T17:16:51.837851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('P', agg='stddev')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.839396Z","iopub.status.idle":"2022-08-04T17:16:51.840366Z","shell.execute_reply.started":"2022-08-04T17:16:51.840090Z","shell.execute_reply":"2022-08-04T17:16:51.840119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('P', agg='min')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.843305Z","iopub.status.idle":"2022-08-04T17:16:51.843917Z","shell.execute_reply.started":"2022-08-04T17:16:51.843648Z","shell.execute_reply":"2022-08-04T17:16:51.843676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('P', agg='max')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.846008Z","iopub.status.idle":"2022-08-04T17:16:51.846764Z","shell.execute_reply.started":"2022-08-04T17:16:51.846345Z","shell.execute_reply":"2022-08-04T17:16:51.846375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('P', agg='last_value')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.849035Z","iopub.status.idle":"2022-08-04T17:16:51.849992Z","shell.execute_reply.started":"2022-08-04T17:16:51.849750Z","shell.execute_reply":"2022-08-04T17:16:51.849773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Balance Variables","metadata":{}},{"cell_type":"code","source":"plot_histograms('B', agg='growth')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.851146Z","iopub.status.idle":"2022-08-04T17:16:51.851887Z","shell.execute_reply.started":"2022-08-04T17:16:51.851682Z","shell.execute_reply":"2022-08-04T17:16:51.851704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('B', agg='avg')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.853131Z","iopub.status.idle":"2022-08-04T17:16:51.853864Z","shell.execute_reply.started":"2022-08-04T17:16:51.853662Z","shell.execute_reply":"2022-08-04T17:16:51.853684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('B', agg='stddev')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.855042Z","iopub.status.idle":"2022-08-04T17:16:51.855780Z","shell.execute_reply.started":"2022-08-04T17:16:51.855590Z","shell.execute_reply":"2022-08-04T17:16:51.855611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('B', agg='min')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.857029Z","iopub.status.idle":"2022-08-04T17:16:51.857701Z","shell.execute_reply.started":"2022-08-04T17:16:51.857482Z","shell.execute_reply":"2022-08-04T17:16:51.857503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('B', agg='max')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.858894Z","iopub.status.idle":"2022-08-04T17:16:51.859727Z","shell.execute_reply.started":"2022-08-04T17:16:51.859540Z","shell.execute_reply":"2022-08-04T17:16:51.859560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('B', agg='last_value')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.860782Z","iopub.status.idle":"2022-08-04T17:16:51.861642Z","shell.execute_reply.started":"2022-08-04T17:16:51.861436Z","shell.execute_reply":"2022-08-04T17:16:51.861458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Risk Variables","metadata":{}},{"cell_type":"code","source":"plot_histograms('R', agg='growth')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.862981Z","iopub.status.idle":"2022-08-04T17:16:51.863395Z","shell.execute_reply.started":"2022-08-04T17:16:51.863201Z","shell.execute_reply":"2022-08-04T17:16:51.863221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('R', agg='avg')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.864586Z","iopub.status.idle":"2022-08-04T17:16:51.865008Z","shell.execute_reply.started":"2022-08-04T17:16:51.864776Z","shell.execute_reply":"2022-08-04T17:16:51.864793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('R', agg='stddev')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.866018Z","iopub.status.idle":"2022-08-04T17:16:51.866365Z","shell.execute_reply.started":"2022-08-04T17:16:51.866203Z","shell.execute_reply":"2022-08-04T17:16:51.866220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('R', agg='min')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.868030Z","iopub.status.idle":"2022-08-04T17:16:51.868400Z","shell.execute_reply.started":"2022-08-04T17:16:51.868228Z","shell.execute_reply":"2022-08-04T17:16:51.868246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('R', agg='max')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.869628Z","iopub.status.idle":"2022-08-04T17:16:51.870039Z","shell.execute_reply.started":"2022-08-04T17:16:51.869830Z","shell.execute_reply":"2022-08-04T17:16:51.869849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histograms('R', agg='last_value')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T17:16:51.871635Z","iopub.status.idle":"2022-08-04T17:16:51.872235Z","shell.execute_reply.started":"2022-08-04T17:16:51.872045Z","shell.execute_reply":"2022-08-04T17:16:51.872066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}