{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-02T04:52:29.273081Z","iopub.execute_input":"2022-07-02T04:52:29.274286Z","iopub.status.idle":"2022-07-02T04:52:29.312013Z","shell.execute_reply.started":"2022-07-02T04:52:29.274152Z","shell.execute_reply":"2022-07-02T04:52:29.311232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        file_size = round(os.path.getsize(os.path.join(dirname, filename)) / (1e9), 2)\n        print(f\"Filename : {filename} \\t File Size : {file_size} GB\")","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:52:29.315020Z","iopub.execute_input":"2022-07-02T04:52:29.315983Z","iopub.status.idle":"2022-07-02T04:52:29.327403Z","shell.execute_reply.started":"2022-07-02T04:52:29.315943Z","shell.execute_reply":"2022-07-02T04:52:29.326392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as mp\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport warnings\nimport random\nfrom tqdm import tqdm\n\nrandom.seed(42)\nwarnings.filterwarnings('ignore')\nsn.color_palette(\"flare\", as_cmap=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:52:29.329249Z","iopub.execute_input":"2022-07-02T04:52:29.329566Z","iopub.status.idle":"2022-07-02T04:52:30.531963Z","shell.execute_reply.started":"2022-07-02T04:52:29.329535Z","shell.execute_reply":"2022-07-02T04:52:30.531096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:52:30.533603Z","iopub.execute_input":"2022-07-02T04:52:30.534084Z","iopub.status.idle":"2022-07-02T04:52:46.783042Z","shell.execute_reply.started":"2022-07-02T04:52:30.534051Z","shell.execute_reply":"2022-07-02T04:52:46.781593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ndf_train = df_train.merge(labels, left_on = 'customer_ID', right_on='customer_ID')","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:52:46.784465Z","iopub.execute_input":"2022-07-02T04:52:46.784697Z","iopub.status.idle":"2022-07-02T04:54:45.399522Z","shell.execute_reply.started":"2022-07-02T04:52:46.784651Z","shell.execute_reply":"2022-07-02T04:54:45.398314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:45.403022Z","iopub.execute_input":"2022-07-02T04:54:45.403373Z","iopub.status.idle":"2022-07-02T04:54:45.410354Z","shell.execute_reply.started":"2022-07-02T04:54:45.403345Z","shell.execute_reply":"2022-07-02T04:54:45.409173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:45.411830Z","iopub.execute_input":"2022-07-02T04:54:45.412671Z","iopub.status.idle":"2022-07-02T04:54:45.443863Z","shell.execute_reply.started":"2022-07-02T04:54:45.412637Z","shell.execute_reply":"2022-07-02T04:54:45.442965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_values = df_train.isna().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:45.445164Z","iopub.execute_input":"2022-07-02T04:54:45.445582Z","iopub.status.idle":"2022-07-02T04:54:47.582546Z","shell.execute_reply.started":"2022-07-02T04:54:45.445546Z","shell.execute_reply":"2022-07-02T04:54:47.581649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_values","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:47.583638Z","iopub.execute_input":"2022-07-02T04:54:47.583932Z","iopub.status.idle":"2022-07-02T04:54:47.591294Z","shell.execute_reply.started":"2022-07-02T04:54:47.583906Z","shell.execute_reply":"2022-07-02T04:54:47.590241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"Distribution of null\")\nnull_values[null_values>0].plot(kind='hist')","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:47.592514Z","iopub.execute_input":"2022-07-02T04:54:47.592831Z","iopub.status.idle":"2022-07-02T04:54:47.823284Z","shell.execute_reply.started":"2022-07-02T04:54:47.592804Z","shell.execute_reply":"2022-07-02T04:54:47.821940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(40,12))\nplt.title(\"Null counts\")\nplt.xlabel(\"Cols\")\nplt.ylabel(\"Count\")\nnull_values[null_values>0].plot(kind=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:47.824766Z","iopub.execute_input":"2022-07-02T04:54:47.825038Z","iopub.status.idle":"2022-07-02T04:54:48.468110Z","shell.execute_reply.started":"2022-07-02T04:54:47.825011Z","shell.execute_reply":"2022-07-02T04:54:48.466840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sn.countplot(df_train[\"target\"].values).set_xlabel(\"Target\");","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:48.471597Z","iopub.execute_input":"2022-07-02T04:54:48.471881Z","iopub.status.idle":"2022-07-02T04:54:49.112343Z","shell.execute_reply.started":"2022-07-02T04:54:48.471854Z","shell.execute_reply":"2022-07-02T04:54:49.111637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"number of unique customer:\",len(df_train[\"customer_ID\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:49.113450Z","iopub.execute_input":"2022-07-02T04:54:49.114015Z","iopub.status.idle":"2022-07-02T04:54:49.823898Z","shell.execute_reply.started":"2022-07-02T04:54:49.113985Z","shell.execute_reply":"2022-07-02T04:54:49.822396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cust_id = np.random.choice(df_train[\"customer_ID\"])\ndf_train[df_train[\"customer_ID\"]==cust_id]","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:54:49.825239Z","iopub.execute_input":"2022-07-02T04:54:49.825518Z","iopub.status.idle":"2022-07-02T04:54:51.441083Z","shell.execute_reply.started":"2022-07-02T04:54:49.825490Z","shell.execute_reply":"2022-07-02T04:54:51.439613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train[\"customer_ID\"]== cust_id][\"S_2\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:55:07.060051Z","iopub.execute_input":"2022-07-02T04:55:07.060411Z","iopub.status.idle":"2022-07-02T04:55:07.754187Z","shell.execute_reply.started":"2022-07-02T04:55:07.060382Z","shell.execute_reply":"2022-07-02T04:55:07.753521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_customers =np.unique(df_train[\"customer_ID\"])[:100]\nid_count=df_train[df_train[\"customer_ID\"].isin(rand_customers)].groupby(\"customer_ID\").agg(\"count\")","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:55:11.527569Z","iopub.execute_input":"2022-07-02T04:55:11.528069Z","iopub.status.idle":"2022-07-02T04:55:18.998489Z","shell.execute_reply.started":"2022-07-02T04:55:11.528043Z","shell.execute_reply":"2022-07-02T04:55:18.997480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(25,15))\nid_count[\"S_2\"].plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:55:33.232410Z","iopub.execute_input":"2022-07-02T04:55:33.232723Z","iopub.status.idle":"2022-07-02T04:55:35.824365Z","shell.execute_reply.started":"2022-07-02T04:55:33.232694Z","shell.execute_reply":"2022-07-02T04:55:35.823026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Correlation of each variable with the target","metadata":{}},{"cell_type":"code","source":"var_count={}\nfor col in df_train.columns:\n    if col.startswith(\"S_\"):\n        var_count[\"Spend variables\"] =var_count.get(\"Spend variables\",0)+1\n    if col.startswith(\"D_\"):\n        var_count[\"Deliquency variables\"]=var_count.get(\"Deliquency variables\",0)+1\n    if col.startswith(\"B_\"):\n        var_count[\"Balance variables\"]=var_count.get(\"Balance variables\",0)+1\n    if col.startswith(\"R_\"):\n        var_count[\"Risk variables\"]=var_count.get(\"Risk variables\",0)+1\n    if col.startswith(\"P_\"):\n        var_count[\"Payment variables\"]=var_count.get(\"Payment variables\",0)+1\nplt.figure(figsize=(16,16))\nsn.barplot(x=list(var_count.keys()), y = list(var_count.values()))\n        \n    \n","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:55:48.823740Z","iopub.execute_input":"2022-07-02T04:55:48.824100Z","iopub.status.idle":"2022-07-02T04:55:48.997896Z","shell.execute_reply.started":"2022-07-02T04:55:48.824070Z","shell.execute_reply":"2022-07-02T04:55:48.996743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"payment_var = [col for col in df_train.columns if col.startswith(\"P_\")]\ncorr = df_train[payment_var+[\"target\"]].corr()\nsn.heatmap(corr,annot = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:56:00.487620Z","iopub.execute_input":"2022-07-02T04:56:00.487950Z","iopub.status.idle":"2022-07-02T04:56:01.134374Z","shell.execute_reply.started":"2022-07-02T04:56:00.487925Z","shell.execute_reply":"2022-07-02T04:56:01.133100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1,3, figsize=(20,5))\naxes = axes.ravel()\n\nfor i, col in enumerate(payment_var)  :\n    sn.histplot(data = df_train, x = col, hue='target', ax=axes[i])\n\nfig.suptitle(\"Distribution of Payment Variables w.r.t target\")\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:56:10.223236Z","iopub.execute_input":"2022-07-02T04:56:10.223612Z","iopub.status.idle":"2022-07-02T04:56:30.941880Z","shell.execute_reply.started":"2022-07-02T04:56:10.223583Z","shell.execute_reply":"2022-07-02T04:56:30.940633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"P_4\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:56:30.944200Z","iopub.execute_input":"2022-07-02T04:56:30.944534Z","iopub.status.idle":"2022-07-02T04:56:31.110263Z","shell.execute_reply.started":"2022-07-02T04:56:30.944502Z","shell.execute_reply":"2022-07-02T04:56:31.109121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"P_4\"]=df_train[\"P_4\"].apply(lambda x:0 if x==0 else 1)\nplt.title(\"P_4 with respect to Target\")\nsn.countplot(data=df_train, x=\"P_4\", hue= \"target\")","metadata":{"execution":{"iopub.status.busy":"2022-07-02T04:56:31.111773Z","iopub.execute_input":"2022-07-02T04:56:31.112441Z","iopub.status.idle":"2022-07-02T04:56:35.249578Z","shell.execute_reply.started":"2022-07-02T04:56:31.112408Z","shell.execute_reply":"2022-07-02T04:56:35.248361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{}}]}