{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport sys\n\nimport matplotlib.pyplot as plt \nimport seaborn as sns\n\nimport warnings\nwarnings.simplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:21.043695Z","iopub.execute_input":"2022-07-03T05:34:21.044114Z","iopub.status.idle":"2022-07-03T05:34:21.051331Z","shell.execute_reply.started":"2022-07-03T05:34:21.044083Z","shell.execute_reply":"2022-07-03T05:34:21.050065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First of all, let's see which files we have:\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:21.120360Z","iopub.execute_input":"2022-07-03T05:34:21.121349Z","iopub.status.idle":"2022-07-03T05:34:21.134120Z","shell.execute_reply.started":"2022-07-03T05:34:21.121313Z","shell.execute_reply":"2022-07-03T05:34:21.132879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#555555;border-radius:5px;color:#FFFFFF'>Reading Data</div></b>","metadata":{}},{"cell_type":"code","source":"%%time\n\n# Since the train and test files are too large to read in csv format, we will read the data from the parquet files\n# thanks to [raddar](https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format)\n\ndf_train = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:21.208495Z","iopub.execute_input":"2022-07-03T05:34:21.208894Z","iopub.status.idle":"2022-07-03T05:34:39.822276Z","shell.execute_reply.started":"2022-07-03T05:34:21.208860Z","shell.execute_reply":"2022-07-03T05:34:39.820775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The \"S_2\" column seems as the date column. Let's change its type to datetime:\n\ndf_train[\"S_2\"] = pd.to_datetime(df_train[\"S_2\"])\ndf_train[\"S_2\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:39.825781Z","iopub.execute_input":"2022-07-03T05:34:39.826274Z","iopub.status.idle":"2022-07-03T05:34:40.903245Z","shell.execute_reply.started":"2022-07-03T05:34:39.826239Z","shell.execute_reply":"2022-07-03T05:34:40.901770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Let's group the data by customer_ID and take the most recent record of each customer:\n\ndf_train = df_train.groupby([\"customer_ID\"]).tail(1).set_index(\"customer_ID\")","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:40.904761Z","iopub.execute_input":"2022-07-03T05:34:40.905160Z","iopub.status.idle":"2022-07-03T05:34:43.332579Z","shell.execute_reply.started":"2022-07-03T05:34:40.905125Z","shell.execute_reply":"2022-07-03T05:34:43.331070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:43.335650Z","iopub.execute_input":"2022-07-03T05:34:43.336029Z","iopub.status.idle":"2022-07-03T05:34:43.644899Z","shell.execute_reply.started":"2022-07-03T05:34:43.335998Z","shell.execute_reply":"2022-07-03T05:34:43.643713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Let's read the train_labels.csv file which contains the target values.\n\ndf_train_labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\")\ndf_train_labels[\"target\"] = df_train_labels[\"target\"].astype(\"int8\")","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:43.646237Z","iopub.execute_input":"2022-07-03T05:34:43.646582Z","iopub.status.idle":"2022-07-03T05:34:44.399110Z","shell.execute_reply.started":"2022-07-03T05:34:43.646552Z","shell.execute_reply":"2022-07-03T05:34:44.397618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_train = df_train.merge(df_train_labels, on=\"customer_ID\", how='left')\ndel df_train_labels\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:44.400683Z","iopub.execute_input":"2022-07-03T05:34:44.401156Z","iopub.status.idle":"2022-07-03T05:34:46.144626Z","shell.execute_reply.started":"2022-07-03T05:34:44.401122Z","shell.execute_reply":"2022-07-03T05:34:46.142620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:46.146489Z","iopub.execute_input":"2022-07-03T05:34:46.146842Z","iopub.status.idle":"2022-07-03T05:34:46.173355Z","shell.execute_reply.started":"2022-07-03T05:34:46.146811Z","shell.execute_reply":"2022-07-03T05:34:46.172005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#555555;border-radius:5px;;color:#FFFFFF'>Null Values</div></b>","metadata":{}},{"cell_type":"code","source":"%%time\n\n# null values of each column at different target values:\n\ndf_nulls = pd.DataFrame(index=df_train.columns)\nt0=[]\nt1=[]\ndf0=df_train[df_train[\"target\"]==0]\ndf1=df_train[df_train[\"target\"]==1]\n\nfor col in df_train.columns:\n    t0.append(len(df0[df0[col].isnull()]))\n    t1.append(len(df1[df1[col].isnull()]))\n\ndf_nulls[\"t0\"] = t0\ndf_nulls[\"t1\"] = t1\ndf_nulls","metadata":{"execution":{"iopub.status.busy":"2022-07-03T06:19:50.610004Z","iopub.execute_input":"2022-07-03T06:19:50.610460Z","iopub.status.idle":"2022-07-03T06:19:54.906329Z","shell.execute_reply.started":"2022-07-03T06:19:50.610423Z","shell.execute_reply":"2022-07-03T06:19:54.905004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#555555;border-radius:5px;;color:#FFFFFF'>Correlation of columns</div></b>","metadata":{}},{"cell_type":"code","source":"%%time\ncorr = df_train.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T05:34:46.174759Z","iopub.execute_input":"2022-07-03T05:34:46.175802Z","iopub.status.idle":"2022-07-03T05:35:33.185431Z","shell.execute_reply.started":"2022-07-03T05:34:46.175759Z","shell.execute_reply":"2022-07-03T05:35:33.181363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#555555;border-radius:5px;color:#FFFFFF'>int8 Type Columns</div></b>","metadata":{}},{"cell_type":"code","source":"%%time\nfig, ax = plt.subplots(int(np.ceil(len(df_train[df_train.columns[df_train.dtypes == \"int8\"]].columns)/3)),3,\n                       figsize=(20,4*np.ceil(len(df_train[df_train.columns[df_train.dtypes == \"int8\"]].columns)/3)))\nn=0\nfor col in df_train.columns[df_train.dtypes == \"int8\"]:\n    s = sns.countplot(data=df_train[[col]],x=col,ax=ax[int(n/3), np.mod(n,3)], hue=df_train[\"target\"])\n    for index, label in enumerate(s.get_xticklabels()):\n        if df_train[col].max()>100:\n            if index % 50 == 0:\n                label.set_visible(True)\n            else:\n                label.set_visible(False)\n        elif df_train[col].max()>10:\n            if index % 5 == 0:\n                label.set_visible(True)\n            else:\n                label.set_visible(False)\n                \n    ax[int(n/3), np.mod(n,3)].set_title(\"(Note: correlation between \"+col+\" and target is \"+str(np.round(corr.loc[\"target\",col],2))+\"\\n\"+\n                                       \"null count at target=0: \"+str(df_nulls.loc[col,\"t0\"])+\n                                        \", at target=1: \"+str(df_nulls.loc[col,\"t1\"])+\")\")\n    n+=1\nplt.subplots_adjust(hspace=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T06:31:31.105246Z","iopub.execute_input":"2022-07-03T06:31:31.105691Z","iopub.status.idle":"2022-07-03T06:32:10.092683Z","shell.execute_reply.started":"2022-07-03T06:31:31.105655Z","shell.execute_reply":"2022-07-03T06:32:10.091522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#555555;border-radius:5px;color:#FFFFFF'>int16 Type Columns</div></b>","metadata":{}},{"cell_type":"code","source":"%%time\n\nfig, ax = plt.subplots(int(np.ceil(len(df_train[df_train.columns[df_train.dtypes == \"int16\"]].columns)/3)),3,\n                       figsize=(20,4*np.ceil(len(df_train[df_train.columns[df_train.dtypes == \"int16\"]].columns)/3)))\nn=0\nfor col in df_train.columns[df_train.dtypes == \"int16\"]:\n    s = sns.countplot(data=df_train[[col]],x=col,ax=ax[int(n/3), np.mod(n,3)], hue=df_train[\"target\"])\n    for index, label in enumerate(s.get_xticklabels()):\n        if df_train[col].max()>100:\n            if index % 50 == 0:\n                label.set_visible(True)\n            else:\n                label.set_visible(False)\n        elif df_train[col].max()>10:\n            if index % 5 == 0:\n                label.set_visible(True)\n            else:\n                label.set_visible(False)\n                \n    ax[int(n/3), np.mod(n,3)].set_title(\"(Note: correlation between \"+col+\" and target is \"+str(np.round(corr.loc[\"target\",col],2))+\"\\n\"+\n                                       \"null count at target=0: \"+str(df_nulls.loc[col,\"t0\"])+\n                                        \", at target=1: \"+str(df_nulls.loc[col,\"t1\"])+\")\")\n    n+=1\nplt.subplots_adjust(hspace=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T06:33:54.005743Z","iopub.execute_input":"2022-07-03T06:33:54.006250Z","iopub.status.idle":"2022-07-03T06:34:09.574027Z","shell.execute_reply.started":"2022-07-03T06:33:54.006213Z","shell.execute_reply":"2022-07-03T06:34:09.572508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#555555;border-radius:5px;color:#FFFFFF'>float32 Type Columns</div></b>","metadata":{}},{"cell_type":"code","source":"%%time\nfig, ax = plt.subplots(int(np.ceil(len(df_train[df_train.columns[df_train.dtypes == \"float32\"]].columns)/3)),3,\n                       figsize=(20,4*np.ceil(len(df_train[df_train.columns[df_train.dtypes == \"float32\"]].columns)/3)))\nn=0\nfor col in df_train.columns[df_train.dtypes == \"float32\"]:\n    \n    s = sns.histplot(data=df_train[[col]], x=col, ax=ax[int(n/3), np.mod(n,3)], bins=32, hue=df_train[\"target\"],multiple=\"dodge\")\n    \n    ax[int(n/3), np.mod(n,3)].set_title(\"(Note: correlation between \"+col+\" and target is \"+str(np.round(corr.loc[\"target\",col],2))+\"\\n\"+\n                                       \"null count at target=0: \"+str(df_nulls.loc[col,\"t0\"])+\n                                        \", at target=1: \"+str(df_nulls.loc[col,\"t1\"])+\")\")\n    n+=1\n\nplt.subplots_adjust(hspace=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-03T06:34:15.742817Z","iopub.execute_input":"2022-07-03T06:34:15.743274Z","iopub.status.idle":"2022-07-03T06:35:02.282473Z","shell.execute_reply.started":"2022-07-03T06:34:15.743239Z","shell.execute_reply":"2022-07-03T06:35:02.281481Z"},"trusted":true},"execution_count":null,"outputs":[]}]}