{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.100796Z","iopub.status.idle":"2022-08-08T19:08:49.101207Z","shell.execute_reply.started":"2022-08-08T19:08:49.101027Z","shell.execute_reply":"2022-08-08T19:08:49.101047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pandas\nimport pandas as pd\npd.set_option(\"display.max_columns\", 1000)\npd.set_option(\"display.max_rows\", 1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.111290Z","iopub.execute_input":"2022-08-08T19:08:49.111979Z","iopub.status.idle":"2022-08-08T19:08:49.118484Z","shell.execute_reply.started":"2022-08-08T19:08:49.111912Z","shell.execute_reply":"2022-08-08T19:08:49.117069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotly\nimport plotly.io as pio\nimport plotly.graph_objects as go\npio.templates[\"draft\"] = go.layout.Template(\n    layout_annotations=[\n        dict(\n            textangle=-30,\n            opacity=0.1,\n            font=dict(color=\"black\", size=100),\n            xref=\"paper\",\n            yref=\"paper\",\n            x=0.5,\n            y=0.5,\n            showarrow=False,\n        )\n    ]\n)\npio.templates.default = \"draft\"","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.155769Z","iopub.execute_input":"2022-08-08T19:08:49.157883Z","iopub.status.idle":"2022-08-08T19:08:49.258078Z","shell.execute_reply.started":"2022-08-08T19:08:49.157829Z","shell.execute_reply":"2022-08-08T19:08:49.256982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.259919Z","iopub.execute_input":"2022-08-08T19:08:49.260353Z","iopub.status.idle":"2022-08-08T19:08:49.265367Z","shell.execute_reply.started":"2022-08-08T19:08:49.260314Z","shell.execute_reply":"2022-08-08T19:08:49.264165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.267307Z","iopub.execute_input":"2022-08-08T19:08:49.267877Z","iopub.status.idle":"2022-08-08T19:08:49.280934Z","shell.execute_reply.started":"2022-08-08T19:08:49.267822Z","shell.execute_reply":"2022-08-08T19:08:49.279883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading Data","metadata":{}},{"cell_type":"markdown","source":"From the Competition Data Description:<br><br>\n<b>D_* = Delinquency variables<br>\nS_* = Spend variables<br>\nP_* = Payment variables<br>\nB_* = Balance variables<br>\nR_* = Risk variables</b><br><br>\nWith the following features being categorical:<br>\n<b>['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']</b>","metadata":{}},{"cell_type":"markdown","source":"## Getting Optimal Column Types","metadata":{}},{"cell_type":"markdown","source":"We'll only read few rows from the dataset to check columns' types.","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", nrows=500)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.368696Z","iopub.execute_input":"2022-08-08T19:08:49.369789Z","iopub.status.idle":"2022-08-08T19:08:49.585390Z","shell.execute_reply.started":"2022-08-08T19:08:49.369731Z","shell.execute_reply":"2022-08-08T19:08:49.584533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_columns = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nnon_cat_columns = set(df.columns) - set(cat_columns)\nd_columns = [col for col in non_cat_columns if \"D_\" in col]\ns_columns = [col for col in non_cat_columns if \"S_\" in col]\np_columns = [col for col in non_cat_columns if \"P_\" in col]\nr_columns = [col for col in non_cat_columns if \"R_\" in col]\nb_columns = [col for col in non_cat_columns if \"B_\" in col]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.586996Z","iopub.execute_input":"2022-08-08T19:08:49.587385Z","iopub.status.idle":"2022-08-08T19:08:49.594944Z","shell.execute_reply.started":"2022-08-08T19:08:49.587350Z","shell.execute_reply":"2022-08-08T19:08:49.593764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[b_columns].dtypes # all float64 except B_31 int64\n# df[r_columns].dtypes # all float64\n# df[p_columns].dtypes # all float 64\n# df[s_columns].dtypes # all float64 except S_2 object\ns_columns.remove(\"S_2\")\n# df[d_columns].dtypes # all float64\nfloat_cols = set([*d_columns, *s_columns, *p_columns, *r_columns, *b_columns]) - {\"B_31\", \"S_2\"}","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.596720Z","iopub.execute_input":"2022-08-08T19:08:49.597343Z","iopub.status.idle":"2022-08-08T19:08:49.607360Z","shell.execute_reply.started":"2022-08-08T19:08:49.597305Z","shell.execute_reply":"2022-08-08T19:08:49.606480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"B_31 Unique Values: {df['B_31'].unique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.609003Z","iopub.execute_input":"2022-08-08T19:08:49.609793Z","iopub.status.idle":"2022-08-08T19:08:49.623722Z","shell.execute_reply.started":"2022-08-08T19:08:49.609755Z","shell.execute_reply":"2022-08-08T19:08:49.622751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_columns:\n    print(f\"{col} Unique Values: {df[col].unique()} ---- Column Type: {df[col].dtypes}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.625268Z","iopub.execute_input":"2022-08-08T19:08:49.625673Z","iopub.status.idle":"2022-08-08T19:08:49.642071Z","shell.execute_reply.started":"2022-08-08T19:08:49.625636Z","shell.execute_reply":"2022-08-08T19:08:49.640917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<b>Using float16 dtype with Pandas isn't recommended, but we'll set this type to be able to read data easily then we can adjust when manipulating the dataset.</b><br>\nGithub Issue for float16 with Pandas: [ https://github.com/pandas-dev/pandas/issues/9220](http://)","metadata":{}},{"cell_type":"code","source":"dtypes_optimal = {col:\"float16\"for col in float_cols}\n\nint_cat_cols = list(set(cat_columns) - {\"D_63\", \"D_64\"})\ndtypes_optimal[\"B_31\"] =  pd.Int8Dtype()\nfor col in cat_columns:\n    dtypes_optimal[col] = pd.Int8Dtype() if col in int_cat_cols else \"category\"  \n# dtypes_optimal[\"customer_ID\"] = \"object\"\n# dtypes_optimal","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.654651Z","iopub.execute_input":"2022-08-08T19:08:49.655294Z","iopub.status.idle":"2022-08-08T19:08:49.661932Z","shell.execute_reply.started":"2022-08-08T19:08:49.655245Z","shell.execute_reply":"2022-08-08T19:08:49.660973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.702517Z","iopub.execute_input":"2022-08-08T19:08:49.703251Z","iopub.status.idle":"2022-08-08T19:08:49.848603Z","shell.execute_reply.started":"2022-08-08T19:08:49.703202Z","shell.execute_reply":"2022-08-08T19:08:49.847361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading Train Data","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", dtype=dtypes_optimal)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:08:49.850877Z","iopub.execute_input":"2022-08-08T19:08:49.851544Z","iopub.status.idle":"2022-08-08T19:10:23.333448Z","shell.execute_reply.started":"2022-08-08T19:08:49.851479Z","shell.execute_reply":"2022-08-08T19:10:23.330965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\", dtype={\"target\": np.int8})","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.334791Z","iopub.status.idle":"2022-08-08T19:10:23.335453Z","shell.execute_reply.started":"2022-08-08T19:10:23.335246Z","shell.execute_reply":"2022-08-08T19:10:23.335268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mem_usage = round(train_data.memory_usage().sum()/(1024**3), 3)\nprint(f\"Train Data Memory Usage: {mem_usage}GB\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.336788Z","iopub.status.idle":"2022-08-08T19:10:23.337629Z","shell.execute_reply.started":"2022-08-08T19:10:23.337409Z","shell.execute_reply":"2022-08-08T19:10:23.337430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Target Dataframe Memory Usage: {round(train_labels.memory_usage().sum()/(1024**2), 2)}MB\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.338574Z","iopub.status.idle":"2022-08-08T19:10:23.339389Z","shell.execute_reply.started":"2022-08-08T19:10:23.339186Z","shell.execute_reply":"2022-08-08T19:10:23.339207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = pd.merge(train_data, train_labels, on=\"customer_ID\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.340488Z","iopub.status.idle":"2022-08-08T19:10:23.341185Z","shell.execute_reply.started":"2022-08-08T19:10:23.340977Z","shell.execute_reply":"2022-08-08T19:10:23.340999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Data","metadata":{"execution":{"iopub.status.busy":"2022-05-29T20:24:40.424994Z","iopub.execute_input":"2022-05-29T20:24:40.425379Z","iopub.status.idle":"2022-05-29T20:24:40.429459Z","shell.execute_reply.started":"2022-05-29T20:24:40.425347Z","shell.execute_reply":"2022-05-29T20:24:40.428483Z"}}},{"cell_type":"markdown","source":"## NaN Values Processing","metadata":{"execution":{"iopub.status.busy":"2022-08-06T19:41:32.958951Z","iopub.execute_input":"2022-08-06T19:41:32.959490Z","iopub.status.idle":"2022-08-06T19:41:32.964394Z","shell.execute_reply.started":"2022-08-06T19:41:32.959448Z","shell.execute_reply":"2022-08-06T19:41:32.963146Z"}}},{"cell_type":"code","source":"nan_values_pct = 100 * round(train_data.isna().sum() / len(train_data), 4)\nnan_values_pct = nan_values_pct.sort_values(ascending=False)\nnan_values_pct = nan_values_pct[nan_values_pct > 0]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.342657Z","iopub.status.idle":"2022-08-08T19:10:23.343062Z","shell.execute_reply.started":"2022-08-08T19:10:23.342879Z","shell.execute_reply":"2022-08-08T19:10:23.342900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_values_pct.plot(kind=\"bar\", figsize=(25,7), grid=True, title=\"Percentage of NaN Values By Column\", ylabel=\"%\", xlabel=\"Column Name\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.344573Z","iopub.status.idle":"2022-08-08T19:10:23.345126Z","shell.execute_reply.started":"2022-08-08T19:10:23.344925Z","shell.execute_reply":"2022-08-08T19:10:23.344946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotly Alternative\nfig = go.Figure()\n\nfig.add_trace(go.Bar(x=nan_values_pct.index,\n                     y=nan_values_pct.values))\n\nfig.update_layout(title=\"Percentage of NaN Values By Column\", xaxis_title=\"Column Name\", yaxis_title=\"NaN %\")\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.346001Z","iopub.status.idle":"2022-08-08T19:10:23.346824Z","shell.execute_reply.started":"2022-08-08T19:10:23.346628Z","shell.execute_reply":"2022-08-08T19:10:23.346650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing Columns With NaNs Rate Higher Than Threshold\nnan_pct_threshold = 80\nto_remove_cols = list(nan_values_pct[nan_values_pct > nan_pct_threshold].index)\nprint(f\"Columns With NaN Values Rate > {nan_pct_threshold}%: {len(to_remove_cols)} Columns\")\ntrain_data = train_data.drop(columns=to_remove_cols).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.347898Z","iopub.status.idle":"2022-08-08T19:10:23.348667Z","shell.execute_reply.started":"2022-08-08T19:10:23.348444Z","shell.execute_reply":"2022-08-08T19:10:23.348464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Customers Daily Data","metadata":{}},{"cell_type":"code","source":"print(f\"Dataset Contains Information About {train_data.customer_ID.nunique()} Customers\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.349685Z","iopub.status.idle":"2022-08-08T19:10:23.350060Z","shell.execute_reply.started":"2022-08-08T19:10:23.349877Z","shell.execute_reply":"2022-08-08T19:10:23.349895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_data_length = train_data.customer_ID.value_counts()\ndata_days_nb_customers = customer_data_length.value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.351433Z","iopub.status.idle":"2022-08-08T19:10:23.351838Z","shell.execute_reply.started":"2022-08-08T19:10:23.351661Z","shell.execute_reply":"2022-08-08T19:10:23.351681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_days_nb_customers.plot(kind=\"bar\", figsize=(25,7), grid=True, title=\"Number Of Customers By Data Length\",\n                            ylabel=\"Number of Customers\",\n                            xlabel=\"Number of Data Days\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.352977Z","iopub.status.idle":"2022-08-08T19:10:23.353333Z","shell.execute_reply.started":"2022-08-08T19:10:23.353165Z","shell.execute_reply":"2022-08-08T19:10:23.353182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotly Alternative\nfig = go.Figure()\nfig.add_trace(go.Bar(x=data_days_nb_customers.index, y=data_days_nb_customers.values))\nfig.update_layout(title=\"Number Of Customers By Data Length\", xaxis_title=\"Number of Data Days\", yaxis_title=\"Number of Customers\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.354605Z","iopub.status.idle":"2022-08-08T19:10:23.355379Z","shell.execute_reply.started":"2022-08-08T19:10:23.355179Z","shell.execute_reply":"2022-08-08T19:10:23.355200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in [\"D_39\", \"P_2\"]:\n    fig = go.Figure()\n    for customer_id in list(customer_data_length.index[:5]):\n        fig.add_trace(go.Scatter(x=train_data.loc[train_data.customer_ID==customer_id, \"S_2\"],\n                                 y=train_data.loc[train_data.customer_ID==customer_id, col].astype(np.float32),\n                                 name=customer_id))\n    fig.update_layout(title=f\"{col} Value Over Time For Some Customers\", xaxis=dict(title=\"Date\", automargin=True), yaxis_title=col,\n                      legend=dict(orientation=\"h\", y=-0.2))\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.356626Z","iopub.status.idle":"2022-08-08T19:10:23.357051Z","shell.execute_reply.started":"2022-08-08T19:10:23.356865Z","shell.execute_reply":"2022-08-08T19:10:23.356883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keep_last_only = True","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:10:23.358337Z","iopub.status.idle":"2022-08-08T19:10:23.358787Z","shell.execute_reply.started":"2022-08-08T19:10:23.358601Z","shell.execute_reply":"2022-08-08T19:10:23.358621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if keep_last_only:\n    # A groupby will take too much time and memory, we'll try another method.\n    # This may be updated later by keeping information in another way. \n    train_data = train_data.drop_duplicates(subset=[\"customer_ID\"], keep=\"last\")\n    # Drop S_2 column since we don't need it anymore\n    train_data = train_data.drop(columns=[\"S_2\"]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T18:54:52.875702Z","iopub.execute_input":"2022-08-08T18:54:52.876043Z","iopub.status.idle":"2022-08-08T18:54:52.880130Z","shell.execute_reply.started":"2022-08-08T18:54:52.876019Z","shell.execute_reply":"2022-08-08T18:54:52.879397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run this to check that target variable isn't a function of time (S_2) if customer has multiple data rows.\n# train_data.groupby([\"customer_ID\"])[\"target\"].nunique().unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T10:37:19.531187Z","iopub.execute_input":"2022-08-07T10:37:19.531554Z","iopub.status.idle":"2022-08-07T10:37:19.541701Z","shell.execute_reply.started":"2022-08-07T10:37:19.531520Z","shell.execute_reply":"2022-08-07T10:37:19.540915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis (EDA)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T19:25:54.22185Z","iopub.execute_input":"2022-05-30T19:25:54.222361Z","iopub.status.idle":"2022-05-30T19:25:54.227288Z","shell.execute_reply.started":"2022-05-30T19:25:54.222322Z","shell.execute_reply":"2022-05-30T19:25:54.226179Z"}}},{"cell_type":"code","source":"print(f\"The Number Of Customers In Train Data is {len(train_data)}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T18:54:58.683542Z","iopub.execute_input":"2022-08-08T18:54:58.683930Z","iopub.status.idle":"2022-08-08T18:54:58.694537Z","shell.execute_reply.started":"2022-08-08T18:54:58.683906Z","shell.execute_reply":"2022-08-08T18:54:58.691597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(9, 9))\nplt.pie(x=train_labels.target.value_counts().values,\n       labels=train_labels.target.value_counts().index,\n       shadow=True,\n       explode=(0,0.1),\n       startangle=90,\n       textprops={\"fontsize\":16},\n       autopct='%1.2f%%')\nplt.title('Target Variable Distribution', fontsize=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:22.575533Z","iopub.execute_input":"2022-08-08T19:05:22.575882Z","iopub.status.idle":"2022-08-08T19:05:22.692107Z","shell.execute_reply.started":"2022-08-08T19:05:22.575856Z","shell.execute_reply":"2022-08-08T19:05:22.691499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotly Alternative\nfig = go.Figure()\n\nfig.add_trace(go.Pie(labels=train_labels.target.value_counts().index,\n                     values=train_labels.target.value_counts().values,\n                     textfont_size=20, marker=dict(line=dict(color='#000000', width=2)))\n             )\n\nfig.update_layout(title=\"Target Variable Distribution\")\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:01:29.197215Z","iopub.execute_input":"2022-08-08T19:01:29.197596Z","iopub.status.idle":"2022-08-08T19:01:29.206162Z","shell.execute_reply.started":"2022-08-08T19:01:29.197571Z","shell.execute_reply":"2022-08-08T19:01:29.202228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\n\nvar_counts = [len(d_columns), len(s_columns), len(p_columns), len(b_columns), len(r_columns)]\n\nfig.add_trace(go.Bar(x=[\"Delinquency variables\", \"Spend variables\", \"Payment variables\", \"Balance variables\", \"Risk variables\"],\n                     y=var_counts,\n                     text=[str(x) for x in var_counts], textfont=dict(size=20), textposition='auto')\n             )\n\nfig.update_layout(title=\"Variables Count By Category\", xaxis_title=\"Variables Category\", yaxis_title=\"Count\")\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:32.620540Z","iopub.execute_input":"2022-08-08T19:05:32.620893Z","iopub.status.idle":"2022-08-08T19:05:32.630838Z","shell.execute_reply.started":"2022-08-08T19:05:32.620868Z","shell.execute_reply":"2022-08-08T19:05:32.630282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"NB: Since we dropped some columns with high NaN values rates, we should update the different lists of columns## Payment Variables","metadata":{}},{"cell_type":"code","source":"p_columns = list(set(p_columns) - set(to_remove_cols))\ns_columns = list(set(s_columns) - set(to_remove_cols))\nd_columns = list(set(d_columns) - set(to_remove_cols))\nb_columns = list(set(b_columns) - set(to_remove_cols))\nr_columns = list(set(r_columns) - set(to_remove_cols))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:35.615614Z","iopub.execute_input":"2022-08-08T19:05:35.616841Z","iopub.status.idle":"2022-08-08T19:05:35.622437Z","shell.execute_reply.started":"2022-08-08T19:05:35.616701Z","shell.execute_reply":"2022-08-08T19:05:35.621502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"high_skew_reduction_rates = {}","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:36.152942Z","iopub.execute_input":"2022-08-08T19:05:36.153288Z","iopub.status.idle":"2022-08-08T19:05:36.157460Z","shell.execute_reply.started":"2022-08-08T19:05:36.153263Z","shell.execute_reply":"2022-08-08T19:05:36.156611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Payment Variables","metadata":{}},{"cell_type":"code","source":"# Using the columns ith float16 type will generate errors while plotting\nfor p_col in p_columns:\n    train_data[p_col] = train_data[p_col].astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:39.195890Z","iopub.execute_input":"2022-08-08T19:05:39.196269Z","iopub.status.idle":"2022-08-08T19:05:40.328757Z","shell.execute_reply.started":"2022-08-08T19:05:39.196243Z","shell.execute_reply":"2022-08-08T19:05:40.327955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for p_col in set(p_columns) - set(cat_columns):\n    plt.figure(figsize=(25,5))\n    _ = plt.hist(train_data[p_col], bins=1000)\n    plt.title(f\"{p_col} Distribution Histogram\")\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:40.330589Z","iopub.execute_input":"2022-08-08T19:05:40.330935Z","iopub.status.idle":"2022-08-08T19:05:45.309185Z","shell.execute_reply.started":"2022-08-08T19:05:40.330910Z","shell.execute_reply":"2022-08-08T19:05:45.308328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Plotting P_4 Histogram Distribution will make plot unclear, unless you unselect the trace\n# fig = go.Figure()\n\n# for p_col in [\"P_2\", \"P_3\"]:\n#     fig.add_trace(go.Histogram(x=train_data[p_col], name=p_col))\n# fig.update_layout(title=\"Payment Variables Distributions\")\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:47.800723Z","iopub.execute_input":"2022-08-08T19:05:47.801252Z","iopub.status.idle":"2022-08-08T19:05:47.805555Z","shell.execute_reply.started":"2022-08-08T19:05:47.801223Z","shell.execute_reply":"2022-08-08T19:05:47.804989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = go.Figure()\n# fig.add_trace(go.Histogram(x=train_data[\"P_4\"]))\n# fig.update_layout(title=\"P_4 Variable Distributions\")\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:48.023771Z","iopub.execute_input":"2022-08-08T19:05:48.024204Z","iopub.status.idle":"2022-08-08T19:05:48.026699Z","shell.execute_reply.started":"2022-08-08T19:05:48.024180Z","shell.execute_reply":"2022-08-08T19:05:48.026238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data[train_data.P_4==0]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:48.580028Z","iopub.execute_input":"2022-08-08T19:05:48.580365Z","iopub.status.idle":"2022-08-08T19:05:48.583839Z","shell.execute_reply.started":"2022-08-08T19:05:48.580341Z","shell.execute_reply":"2022-08-08T19:05:48.583006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Skewness Checking**","metadata":{}},{"cell_type":"markdown","source":"\"Skewness refers to a distortion or asymmetry that deviates from the symmetrical bell curve, or normal distribution, in a set of data.\nIf the curve is shifted to the left or to the right, it is said to be skewed.\"\n\n[https://www.investopedia.com/terms/s/skewness.asp](http://)","metadata":{}},{"cell_type":"code","source":"def skewness_level(x):\n    if np.abs(x) > 1:\n        return \"high\"\n    elif -0.5 <= x <= 0.5:\n        return \"symmetrical\"\n    else:\n        return \"moderate\"","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:49.900378Z","iopub.execute_input":"2022-08-08T19:05:49.900686Z","iopub.status.idle":"2022-08-08T19:05:49.906621Z","shell.execute_reply.started":"2022-08-08T19:05:49.900663Z","shell.execute_reply":"2022-08-08T19:05:49.905372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_skewness = train_data[p_columns].agg([\"skew\"]).T\np_skewness[\"skewness_level\"] = p_skewness[\"skew\"].apply(lambda x :skewness_level(x))\np_skewness","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:50.227677Z","iopub.execute_input":"2022-08-08T19:05:50.228009Z","iopub.status.idle":"2022-08-08T19:05:50.401135Z","shell.execute_reply.started":"2022-08-08T19:05:50.227984Z","shell.execute_reply":"2022-08-08T19:05:50.400403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewed_p_variables = list(p_skewness.loc[p_skewness.skewness_level==\"high\"].index)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:50.769539Z","iopub.execute_input":"2022-08-08T19:05:50.770734Z","iopub.status.idle":"2022-08-08T19:05:50.775891Z","shell.execute_reply.started":"2022-08-08T19:05:50.770689Z","shell.execute_reply":"2022-08-08T19:05:50.775245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for p_col in skewed_p_variables:\n#     plt.figure(figsize=(25,5))\n#     _ = plt.hist(np.log(train_data[p_col]), bins=1000)\n#     plt.title(f\"Skewed {p_col} Distribution Histogram\")\n#     plt.grid()\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:51.459669Z","iopub.execute_input":"2022-08-08T19:05:51.461047Z","iopub.status.idle":"2022-08-08T19:05:51.464649Z","shell.execute_reply.started":"2022-08-08T19:05:51.460993Z","shell.execute_reply":"2022-08-08T19:05:51.463717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Plot takes some time...\n# skewed_p_variables = list(p_skewness.loc[p_skewness.skewness_level==\"high\"].index)\n# fig = go.Figure()\n# for p_col in skewed_p_variables:\n#     fig.add_trace(go.Histogram(x=np.log(train_data[p_col]), name=p_col))\n# fig.update_layout(title=\"Skewed Payment Variables Log Transformation\")\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:52.131903Z","iopub.execute_input":"2022-08-08T19:05:52.132515Z","iopub.status.idle":"2022-08-08T19:05:52.136641Z","shell.execute_reply.started":"2022-08-08T19:05:52.132487Z","shell.execute_reply":"2022-08-08T19:05:52.135136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All skewness levels now are moderate --> we'll log-transform these features in train data","metadata":{}},{"cell_type":"code","source":"for skew_p_col in skewed_p_variables:\n    train_data[skew_p_col] = np.log(train_data[skew_p_col])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:53.287074Z","iopub.execute_input":"2022-08-08T19:05:53.287949Z","iopub.status.idle":"2022-08-08T19:05:53.335681Z","shell.execute_reply.started":"2022-08-08T19:05:53.287910Z","shell.execute_reply":"2022-08-08T19:05:53.334582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_skewness[\"log_transform_skewness_level\"] = train_data[p_columns].skew().T\np_skewness[\"log_transform_skewness_level\"] = p_skewness[\"log_transform_skewness_level\"].apply(lambda x :skewness_level(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:56.219531Z","iopub.execute_input":"2022-08-08T19:05:56.219879Z","iopub.status.idle":"2022-08-08T19:05:56.479654Z","shell.execute_reply.started":"2022-08-08T19:05:56.219855Z","shell.execute_reply":"2022-08-08T19:05:56.478810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_rate = len(p_skewness.loc[(p_skewness.skewness_level==\"high\") & (p_skewness.log_transform_skewness_level != \"high\")]) / len(p_skewness)\nhigh_skew_reduction_rates[\"PAYMENT\"] = round(100*p_rate,2)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:56.504485Z","iopub.execute_input":"2022-08-08T19:05:56.505166Z","iopub.status.idle":"2022-08-08T19:05:56.511406Z","shell.execute_reply.started":"2022-08-08T19:05:56.505139Z","shell.execute_reply":"2022-08-08T19:05:56.510770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spend variables","metadata":{}},{"cell_type":"code","source":"# Using the columns with float16 type will generate errors while plotting\nfor s_col in s_columns:\n    train_data[s_col] = train_data[s_col].astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:05:57.554918Z","iopub.execute_input":"2022-08-08T19:05:57.555848Z","iopub.status.idle":"2022-08-08T19:06:02.973287Z","shell.execute_reply.started":"2022-08-08T19:05:57.555812Z","shell.execute_reply":"2022-08-08T19:06:02.972466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s_skewness = train_data[s_columns].agg([\"skew\"]).T\ns_skewness[\"skewness_level\"] = s_skewness[\"skew\"].apply(lambda x :skewness_level(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:02.974518Z","iopub.execute_input":"2022-08-08T19:06:02.974815Z","iopub.status.idle":"2022-08-08T19:06:04.036886Z","shell.execute_reply.started":"2022-08-08T19:06:02.974790Z","shell.execute_reply":"2022-08-08T19:06:04.036095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewed_s_variables = list(s_skewness.loc[s_skewness.skewness_level==\"high\"].index)\nfor s_col in skewed_s_variables:\n    train_data[s_col] = np.log(train_data[s_col])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:04.037792Z","iopub.execute_input":"2022-08-08T19:06:04.038718Z","iopub.status.idle":"2022-08-08T19:06:04.453391Z","shell.execute_reply.started":"2022-08-08T19:06:04.038692Z","shell.execute_reply":"2022-08-08T19:06:04.452318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s_skewness[\"log_transform_skewness_level\"] = train_data[s_columns].skew().T\ns_skewness[\"log_transform_skewness_level\"] = s_skewness[\"log_transform_skewness_level\"].apply(lambda x :skewness_level(x))\ns_skewness","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:04.454899Z","iopub.execute_input":"2022-08-08T19:06:04.455117Z","iopub.status.idle":"2022-08-08T19:06:08.431620Z","shell.execute_reply.started":"2022-08-08T19:06:04.455094Z","shell.execute_reply":"2022-08-08T19:06:08.430518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s_rate = len(s_skewness.loc[(s_skewness.skewness_level==\"high\") & (s_skewness.log_transform_skewness_level != \"high\")]) / len(s_skewness)\nhigh_skew_reduction_rates[\"SPEND\"] = round(100*s_rate,2)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:08.432601Z","iopub.execute_input":"2022-08-08T19:06:08.432860Z","iopub.status.idle":"2022-08-08T19:06:08.438457Z","shell.execute_reply.started":"2022-08-08T19:06:08.432836Z","shell.execute_reply":"2022-08-08T19:06:08.437904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Delinquency variables","metadata":{}},{"cell_type":"code","source":"# Using the columns with float16 type will generate errors while plotting\nfor d_col in d_columns:\n    train_data[d_col] = train_data[d_col].astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:08.439363Z","iopub.execute_input":"2022-08-08T19:06:08.439958Z","iopub.status.idle":"2022-08-08T19:06:21.336588Z","shell.execute_reply.started":"2022-08-08T19:06:08.439933Z","shell.execute_reply":"2022-08-08T19:06:21.335609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_skewness = train_data[d_columns].agg([\"skew\"]).T\nd_skewness[\"skewness_level\"] = d_skewness[\"skew\"].apply(lambda x :skewness_level(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:21.337471Z","iopub.execute_input":"2022-08-08T19:06:21.337696Z","iopub.status.idle":"2022-08-08T19:06:25.015528Z","shell.execute_reply.started":"2022-08-08T19:06:21.337673Z","shell.execute_reply":"2022-08-08T19:06:25.013303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skewed_d_variables = list(d_skewness.loc[d_skewness.skewness_level==\"high\"].index)\nfor d_col in skewed_d_variables:\n    train_data[d_col] = np.log(train_data[d_col])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:06:25.016551Z","iopub.execute_input":"2022-08-08T19:06:25.016855Z","iopub.status.idle":"2022-08-08T19:06:26.422093Z","shell.execute_reply.started":"2022-08-08T19:06:25.016830Z","shell.execute_reply":"2022-08-08T19:06:26.419567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_skewness[\"log_transform_skewness_level\"] = train_data[d_columns].skew().T\nd_skewness[\"log_transform_skewness_level\"] = d_skewness[\"log_transform_skewness_level\"].apply(lambda x :skewness_level(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T19:07:11.722628Z","iopub.execute_input":"2022-08-08T19:07:11.723431Z","iopub.status.idle":"2022-08-08T19:07:11.743786Z","shell.execute_reply.started":"2022-08-08T19:07:11.723402Z","shell.execute_reply":"2022-08-08T19:07:11.740655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_rate = len(d_skewness.loc[(d_skewness.skewness_level==\"high\") & (d_skewness.log_transform_skewness_level != \"high\")]) / len(d_skewness)\nhigh_skew_reduction_rates[\"DELIQUENCY\"] = round(100*d_rate,2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Balance variable & Risk variables","metadata":{"execution":{"iopub.status.busy":"2022-06-13T20:18:59.966660Z","iopub.execute_input":"2022-06-13T20:18:59.967108Z","iopub.status.idle":"2022-06-13T20:18:59.972230Z","shell.execute_reply.started":"2022-06-13T20:18:59.967075Z","shell.execute_reply":"2022-06-13T20:18:59.971160Z"}}},{"cell_type":"code","source":"# Using the columns with float16 type will generate errors while plotting\nfor b_col in b_columns:\n    train_data[b_col] = train_data[b_col].astype(\"float32\")\nfor r_col in r_columns:\n    train_data[r_col] = train_data[r_col].astype(\"float32\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b_skewness = train_data[b_columns].agg([\"skew\"]).T\nb_skewness[\"skewness_level\"] = b_skewness[\"skew\"].apply(lambda x :skewness_level(x))\nskewed_b_variables = list(b_skewness.loc[b_skewness.skewness_level==\"high\"].index)\nfor b_col in skewed_b_variables:\n    train_data[b_col] = np.log(train_data[b_col])\nb_skewness[\"log_transform_skewness_level\"] = train_data[b_columns].skew().T\nb_skewness[\"log_transform_skewness_level\"] = b_skewness[\"log_transform_skewness_level\"].apply(lambda x :skewness_level(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r_skewness = train_data[r_columns].agg([\"skew\"]).T\nr_skewness[\"skewness_level\"] = r_skewness[\"skew\"].apply(lambda x :skewness_level(x))\nskewed_r_variables = list(r_skewness.loc[r_skewness.skewness_level==\"high\"].index)\nfor r_col in skewed_r_variables:\n    train_data[r_col] = np.log(train_data[r_col])\nr_skewness[\"log_transform_skewness_level\"] = train_data[r_columns].skew().T\nr_skewness[\"log_transform_skewness_level\"] = r_skewness[\"log_transform_skewness_level\"].apply(lambda x :skewness_level(x))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b_rate = len(b_skewness.loc[(b_skewness.skewness_level==\"high\") & (b_skewness.log_transform_skewness_level != \"high\")]) / len(b_skewness)\nhigh_skew_reduction_rates[\"BALANCE\"] = round(100*b_rate,2)\n\nr_rate = len(r_skewness.loc[(r_skewness.skewness_level==\"high\") & (r_skewness.log_transform_skewness_level != \"high\")]) / len(r_skewness)\nhigh_skew_reduction_rates[\"RISK\"] = round(100*r_rate,2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\n\nfig.add_trace(go.Bar(x=list(high_skew_reduction_rates.keys()),\n                    y=list(high_skew_reduction_rates.values())))\n\nfig.update_layout(title=\"Skewness Reduction Rate\", xaxis_title=\"Variables Category\", yaxis_title=\"Reductoon Rate %\")\n\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation","metadata":{}},{"cell_type":"code","source":"cat_columns = list(set(cat_columns) - set(to_remove_cols))\ncontinuous_columns = list(set(train_data.columns) - set(cat_columns))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_matrix = train_data[continuous_columns].corr()\nmask = np.triu(np.ones_like(corr_matrix))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nheatmap = sns.heatmap(corr_matrix, mask=mask)\nplt.title(\"Continuous Variables Correlation Heatmap\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical Variables","metadata":{}},{"cell_type":"code","source":"fig = go.Figure()\n    \nfor cat_column in cat_columns:\n    cat_col_dist = train_data[cat_column].value_counts()\n    fig.add_trace(go.Bar(x=cat_col_dist.index,  y=cat_col_dist.values,  visible=cat_column==cat_columns[0]))\n    \nfig.update_layout(title=f\"{cat_columns[0]} Variable Distribution\", yaxis_title=\"Count\")\n    \nbuttons = []\nfor cat_column in cat_columns:\n    buttons.append(dict(method=\"update\", label=cat_column,\n                        args=[{\"visible\":[c==cat_column for c in cat_columns]},  {\"title\":f\"{cat_column} Variable Distribution\"}]\n                       ))\n\nfig.update_layout(updatemenus=[{\"buttons\":buttons, \"active\":0, \"showactive\":True, \"direction\":\"left\",  \"x\":1, \"y\":1.35}])\n\nfig.update_xaxes(automargin=True)\n\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}