{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# American Express - Default Prediction - Exploratory Data Analysis\n\nPredict if a customer will default in the future\n\nQuick Exploratory Data Analysis for [American Express - Default Prediction](https://www.kaggle.com/competitions/amex-default-prediction/overview) challenge \n","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:53:55.775449Z","iopub.execute_input":"2022-05-25T18:53:55.775995Z","iopub.status.idle":"2022-05-25T18:53:55.785817Z","shell.execute_reply.started":"2022-05-25T18:53:55.775956Z","shell.execute_reply":"2022-05-25T18:53:55.783794Z"}}},{"cell_type":"markdown","source":"![](https://storage.googleapis.com/kaggle-competitions/kaggle/35332/logos/header.png?t=2022-03-23-01-05-50)\n","metadata":{}},{"cell_type":"markdown","source":"\n\n
\n

Quick Navigation

\n\n* [Overview](#1)\n* [Visualizations](#2)\n* [Modeling](#3)","metadata":{}},{"cell_type":"markdown","source":"\n

Overview

","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sn\n\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:04.600328Z","iopub.execute_input":"2022-05-31T23:29:04.601058Z","iopub.status.idle":"2022-05-31T23:29:05.765719Z","shell.execute_reply.started":"2022-05-31T23:29:04.600964Z","shell.execute_reply":"2022-05-31T23:29:05.764620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DATA_PATH = \"../input/amex-default-prediction/train_data.csv\"\nTRAIN_LABELS_PATH = \"../input/amex-default-prediction/train_labels.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:05.772261Z","iopub.execute_input":"2022-05-31T23:29:05.775558Z","iopub.status.idle":"2022-05-31T23:29:05.782977Z","shell.execute_reply.started":"2022-05-31T23:29:05.775482Z","shell.execute_reply":"2022-05-31T23:29:05.781969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset is so big, that's why we can use chunk loading","metadata":{}},{"cell_type":"code","source":"chunksize = 13000\n\ntrain_df_iter = pd.read_csv(TRAIN_DATA_PATH, chunksize=chunksize)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:05.785013Z","iopub.execute_input":"2022-05-31T23:29:05.786192Z","iopub.status.idle":"2022-05-31T23:29:05.809685Z","shell.execute_reply.started":"2022-05-31T23:29:05.786147Z","shell.execute_reply":"2022-05-31T23:29:05.808851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load only one chunk for EDA With the for loop we can load and process all data","metadata":{}},{"cell_type":"code","source":"# for chunk in train_df_example:\n# process(chunk)\n\ntrain_df_example = train_df_iter.__next__()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:05.812646Z","iopub.execute_input":"2022-05-31T23:29:05.813820Z","iopub.status.idle":"2022-05-31T23:29:06.855272Z","shell.execute_reply.started":"2022-05-31T23:29:05.813765Z","shell.execute_reply":"2022-05-31T23:29:06.854175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Labels","metadata":{}},{"cell_type":"code","source":"train_labels_df = pd.read_csv(TRAIN_LABELS_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:06.857082Z","iopub.execute_input":"2022-05-31T23:29:06.857577Z","iopub.status.idle":"2022-05-31T23:29:07.958682Z","shell.execute_reply.started":"2022-05-31T23:29:06.857505Z","shell.execute_reply":"2022-05-31T23:29:07.957531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at one customer","metadata":{}},{"cell_type":"code","source":"example_customer_id = \"000f1c950ae4e388f44e9ba96dd6334dfa85d8be0416d9d0d30228301f2e4cc4\"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:07.960223Z","iopub.execute_input":"2022-05-31T23:29:07.960698Z","iopub.status.idle":"2022-05-31T23:29:07.966829Z","shell.execute_reply.started":"2022-05-31T23:29:07.960654Z","shell.execute_reply":"2022-05-31T23:29:07.965837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_data_ex = train_df_example[train_df_example[\"customer_ID\"] == example_customer_id]","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:07.967927Z","iopub.execute_input":"2022-05-31T23:29:07.968231Z","iopub.status.idle":"2022-05-31T23:29:07.995888Z","shell.execute_reply.started":"2022-05-31T23:29:07.968205Z","shell.execute_reply":"2022-05-31T23:29:07.994838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_data_ex","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:07.997296Z","iopub.execute_input":"2022-05-31T23:29:07.997718Z","iopub.status.idle":"2022-05-31T23:29:08.050745Z","shell.execute_reply.started":"2022-05-31T23:29:07.997683Z","shell.execute_reply":"2022-05-31T23:29:08.049465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Features are anonymized and normalized, and fall into the following general categories:\n- D_* = Delinquency variables \n- S_* = Spend variables \n- P_* = Payment variables \n- B_* = Balance variables \n- R_* = Risk variables ","metadata":{}},{"cell_type":"code","source":"all_cols = list(customer_data_ex.columns)\nprint(all_cols)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.052730Z","iopub.execute_input":"2022-05-31T23:29:08.053226Z","iopub.status.idle":"2022-05-31T23:29:08.060708Z","shell.execute_reply.started":"2022-05-31T23:29:08.053179Z","shell.execute_reply":"2022-05-31T23:29:08.059686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b_cols = list(filter(lambda x: x.startswith(\"B_\"), all_cols))\nprint(b_cols)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.064137Z","iopub.execute_input":"2022-05-31T23:29:08.066799Z","iopub.status.idle":"2022-05-31T23:29:08.073604Z","shell.execute_reply.started":"2022-05-31T23:29:08.066745Z","shell.execute_reply":"2022-05-31T23:29:08.072573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check if the customer will future payment default","metadata":{}},{"cell_type":"code","source":"train_labels_df[train_labels_df[\"customer_ID\"] == example_customer_id]","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.075161Z","iopub.execute_input":"2022-05-31T23:29:08.076171Z","iopub.status.idle":"2022-05-31T23:29:08.178846Z","shell.execute_reply.started":"2022-05-31T23:29:08.076123Z","shell.execute_reply":"2022-05-31T23:29:08.178079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_data_ex.loc[:, \"S_2\"] = pd.to_datetime(customer_data_ex[\"S_2\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.180060Z","iopub.execute_input":"2022-05-31T23:29:08.180834Z","iopub.status.idle":"2022-05-31T23:29:08.189596Z","shell.execute_reply.started":"2022-05-31T23:29:08.180799Z","shell.execute_reply":"2022-05-31T23:29:08.188744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsn.lineplot(data=customer_data_ex, x=\"S_2\", y=\"P_2\")\nplt.title(\"P_2\", fontsize=16)\nplt.xlabel(\"S_2\", fontsize=14)\nplt.ylabel(\"P_2\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.190870Z","iopub.execute_input":"2022-05-31T23:29:08.191433Z","iopub.status.idle":"2022-05-31T23:29:08.523119Z","shell.execute_reply.started":"2022-05-31T23:29:08.191400Z","shell.execute_reply":"2022-05-31T23:29:08.522008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Visualizations

","metadata":{}},{"cell_type":"markdown","source":"Show only 10 first customer's","metadata":{}},{"cell_type":"code","source":"ex_customer_ids = train_labels_df.iloc[:10][\"customer_ID\"].tolist()\nex_customer_data = train_df_example[train_df_example[\"customer_ID\"].isin(ex_customer_ids)]","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.524907Z","iopub.execute_input":"2022-05-31T23:29:08.525247Z","iopub.status.idle":"2022-05-31T23:29:08.534276Z","shell.execute_reply.started":"2022-05-31T23:29:08.525218Z","shell.execute_reply":"2022-05-31T23:29:08.533075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data = pd.merge(ex_customer_data, train_labels_df.iloc[:10], on=\"customer_ID\")\nex_customer_data[\"S_2\"] = pd.to_datetime(ex_customer_data[\"S_2\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.536703Z","iopub.execute_input":"2022-05-31T23:29:08.537699Z","iopub.status.idle":"2022-05-31T23:29:08.564112Z","shell.execute_reply.started":"2022-05-31T23:29:08.537652Z","shell.execute_reply":"2022-05-31T23:29:08.562788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.565495Z","iopub.execute_input":"2022-05-31T23:29:08.566639Z","iopub.status.idle":"2022-05-31T23:29:08.599708Z","shell.execute_reply.started":"2022-05-31T23:29:08.566593Z","shell.execute_reply":"2022-05-31T23:29:08.598762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How their feature time series look like","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor _, group in ex_customer_data.groupby(\"customer_ID\"):\n sn.lineplot(data=group, x=\"S_2\", y=\"P_2\", label=group[\"target\"].max())\nplt.title(\"P_2\", fontsize=16)\nplt.xlabel(\"S_2\", fontsize=14)\nplt.ylabel(\"P_2\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:08.601274Z","iopub.execute_input":"2022-05-31T23:29:08.601923Z","iopub.status.idle":"2022-05-31T23:29:09.171453Z","shell.execute_reply.started":"2022-05-31T23:29:08.601881Z","shell.execute_reply":"2022-05-31T23:29:09.170247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor _, group in ex_customer_data.groupby(\"customer_ID\"):\n sn.lineplot(data=group, x=\"S_2\", y=\"B_1\", label=group[\"target\"].max())\nplt.title(\"B_1\", fontsize=16)\nplt.xlabel(\"S_2\", fontsize=14)\nplt.ylabel(\"B_1\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:09.172824Z","iopub.execute_input":"2022-05-31T23:29:09.173243Z","iopub.status.idle":"2022-05-31T23:29:09.682084Z","shell.execute_reply.started":"2022-05-31T23:29:09.173210Z","shell.execute_reply":"2022-05-31T23:29:09.681012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor _, group in ex_customer_data.groupby(\"customer_ID\"):\n sn.lineplot(data=group, x=\"S_2\", y=\"B_2\", label=group[\"target\"].max())\nplt.title(\"B_2\", fontsize=16)\nplt.xlabel(\"S_2\", fontsize=14)\nplt.ylabel(\"B_2\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:09.683667Z","iopub.execute_input":"2022-05-31T23:29:09.684202Z","iopub.status.idle":"2022-05-31T23:29:10.192757Z","shell.execute_reply.started":"2022-05-31T23:29:09.684155Z","shell.execute_reply":"2022-05-31T23:29:10.191502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take 1000 customers and show features histograms for target customers and for no-target","metadata":{}},{"cell_type":"code","source":"ex_customer_ids = train_labels_df.iloc[:1000][\"customer_ID\"].tolist()\nex_customer_data = train_df_example[train_df_example[\"customer_ID\"].isin(ex_customer_ids)]\nex_customer_data = pd.merge(ex_customer_data, train_labels_df.iloc[:1000], on=\"customer_ID\")\nex_customer_data[\"S_2\"] = pd.to_datetime(ex_customer_data[\"S_2\"])\n\nex_customer_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:10.194338Z","iopub.execute_input":"2022-05-31T23:29:10.194732Z","iopub.status.idle":"2022-05-31T23:29:10.250804Z","shell.execute_reply.started":"2022-05-31T23:29:10.194697Z","shell.execute_reply":"2022-05-31T23:29:10.249811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have 735 no-target customers and 265 target","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsn.countplot(y=ex_customer_data.groupby(\"customer_ID\")[\"target\"].max())\nplt.title(\"Class distribution\", fontsize=16)\nplt.xlabel(\"count\", fontsize=14)\nplt.ylabel(\"target\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:10.252360Z","iopub.execute_input":"2022-05-31T23:29:10.253139Z","iopub.status.idle":"2022-05-31T23:29:10.413569Z","shell.execute_reply.started":"2022-05-31T23:29:10.253103Z","shell.execute_reply":"2022-05-31T23:29:10.412576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsn.countplot(y=ex_customer_data.groupby(\"customer_ID\")[\"target\"].count())\nplt.title(\"Distribution of the number of records for the client\", fontsize=16)\nplt.xlabel(\"count\", fontsize=14)\nplt.ylabel(\"n_records\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:10.415075Z","iopub.execute_input":"2022-05-31T23:29:10.415536Z","iopub.status.idle":"2022-05-31T23:29:10.624639Z","shell.execute_reply.started":"2022-05-31T23:29:10.415475Z","shell.execute_reply":"2022-05-31T23:29:10.623916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsn.histplot(data=ex_customer_data, x=\"S_2\", bins=100)\nplt.title(\"Distribution of records by time\", fontsize=16)\nplt.xlabel(\"count\", fontsize=14)\nplt.ylabel(\"n_records\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:10.625662Z","iopub.execute_input":"2022-05-31T23:29:10.626092Z","iopub.status.idle":"2022-05-31T23:29:11.025888Z","shell.execute_reply.started":"2022-05-31T23:29:10.626064Z","shell.execute_reply":"2022-05-31T23:29:11.025016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sort_f(x):\n try:\n a, b = x.split(\"_\")\n return a, int(b)\n except:\n return \"0\", 0\n\nall_cols = sorted(all_cols, key=sort_f)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:11.027148Z","iopub.execute_input":"2022-05-31T23:29:11.027875Z","iopub.status.idle":"2022-05-31T23:29:11.033243Z","shell.execute_reply.started":"2022-05-31T23:29:11.027838Z","shell.execute_reply":"2022-05-31T23:29:11.032492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = [\n 'B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', \n 'D_126', 'D_63', 'D_64', 'D_66', 'D_68',\n]","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:11.034550Z","iopub.execute_input":"2022-05-31T23:29:11.034986Z","iopub.status.idle":"2022-05-31T23:29:11.046851Z","shell.execute_reply.started":"2022-05-31T23:29:11.034955Z","shell.execute_reply":"2022-05-31T23:29:11.046071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind = 0\nfor col in categorical_cols:\n if ind % 4 == 0:\n plt.figure(figsize=(16, 3))\n plt.subplot(1, 4, ind % 4 + 1)\n \n sn.countplot(data=ex_customer_data, x=col, hue=\"target\")\n plt.ylabel(\"\")\n \n if ind % 4 == 3:\n plt.show()\n \n ind += 1","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:11.048161Z","iopub.execute_input":"2022-05-31T23:29:11.049545Z","iopub.status.idle":"2022-05-31T23:29:12.690793Z","shell.execute_reply.started":"2022-05-31T23:29:11.049482Z","shell.execute_reply":"2022-05-31T23:29:12.689719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind = 0\nfor col in all_cols:\n if col in [\"S_2\", \"customer_ID\", \"target\"] + categorical_cols:\n continue\n \n if ind % 4 == 0:\n plt.figure(figsize=(16, 4))\n plt.subplot(1, 4, ind % 4 + 1)\n \n sn.histplot(data=ex_customer_data, x=col, hue=\"target\", bins=20)\n plt.ylabel(\"\")\n \n if ind % 4 == 3:\n plt.show()\n \n ind += 1","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:12.692342Z","iopub.execute_input":"2022-05-31T23:29:12.692836Z","iopub.status.idle":"2022-05-31T23:29:58.195272Z","shell.execute_reply.started":"2022-05-31T23:29:12.692794Z","shell.execute_reply":"2022-05-31T23:29:58.194239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data[ex_customer_data[\"target\"] == 0][b_cols[:10]].describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:58.198404Z","iopub.execute_input":"2022-05-31T23:29:58.198788Z","iopub.status.idle":"2022-05-31T23:29:58.265469Z","shell.execute_reply.started":"2022-05-31T23:29:58.198748Z","shell.execute_reply":"2022-05-31T23:29:58.264565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data[ex_customer_data[\"target\"] == 1][b_cols[:10]].describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:58.266558Z","iopub.execute_input":"2022-05-31T23:29:58.266877Z","iopub.status.idle":"2022-05-31T23:29:58.315776Z","shell.execute_reply.started":"2022-05-31T23:29:58.266848Z","shell.execute_reply":"2022-05-31T23:29:58.314857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Modeling

","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:04:36.119274Z","iopub.execute_input":"2022-05-26T12:04:36.119671Z","iopub.status.idle":"2022-05-26T12:04:36.126046Z","shell.execute_reply.started":"2022-05-26T12:04:36.119641Z","shell.execute_reply":"2022-05-26T12:04:36.124613Z"}}},{"cell_type":"markdown","source":"Select baseline features based on the graps above","metadata":{}},{"cell_type":"code","source":"X_cols = [\n \"B_2\", \"B_7\", \"B_18\", \"B_23\", \"B_32\", \"D_48\",\n \"D_55\", \"D_61\", \"D_121\", \"P_2\", \"S_11\",\n \n]","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:29:58.317338Z","iopub.execute_input":"2022-05-31T23:29:58.317974Z","iopub.status.idle":"2022-05-31T23:29:58.323251Z","shell.execute_reply.started":"2022-05-31T23:29:58.317929Z","shell.execute_reply":"2022-05-31T23:29:58.322391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Take a small portion of train data for train baseline","metadata":{}},{"cell_type":"code","source":"chunksize = 1000000\n\ntrain_df_iter = pd.read_csv(TRAIN_DATA_PATH, chunksize=chunksize, usecols=[\"customer_ID\"] + X_cols)\n\n# train_df = train_df_iter.__next__()\n\ntrain_df = pd.DataFrame()\nfor i_chunk, chunk in enumerate(train_df_iter):\n train_df = pd.concat([train_df, chunk])\n print(train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:37:39.331320Z","iopub.execute_input":"2022-05-31T23:37:39.331854Z","iopub.status.idle":"2022-05-31T23:42:02.854087Z","shell.execute_reply.started":"2022-05-31T23:37:39.331816Z","shell.execute_reply":"2022-05-31T23:42:02.852827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create mean and last values for selected features","metadata":{}},{"cell_type":"code","source":"train_df_mean = train_df.groupby(\"customer_ID\")[X_cols].mean().reset_index()\ntrain_df_last = train_df.groupby(\"customer_ID\")[X_cols].last().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:44.355787Z","iopub.execute_input":"2022-05-31T23:43:44.357438Z","iopub.status.idle":"2022-05-31T23:43:48.922545Z","shell.execute_reply.started":"2022-05-31T23:43:44.357379Z","shell.execute_reply":"2022-05-31T23:43:48.921536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(\n left=train_df_mean, \n right=train_df_last, \n how=\"inner\",\n on=\"customer_ID\",\n suffixes=(\"_mean\", \"_last\"),\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:48.924251Z","iopub.execute_input":"2022-05-31T23:43:48.924616Z","iopub.status.idle":"2022-05-31T23:43:49.358614Z","shell.execute_reply.started":"2022-05-31T23:43:48.924585Z","shell.execute_reply":"2022-05-31T23:43:49.357573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(train_df, train_labels_df, on=\"customer_ID\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:49.360183Z","iopub.execute_input":"2022-05-31T23:43:49.361263Z","iopub.status.idle":"2022-05-31T23:43:49.879681Z","shell.execute_reply.started":"2022-05-31T23:43:49.361218Z","shell.execute_reply":"2022-05-31T23:43:49.878666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hard fillna. We need to recheck this","metadata":{}},{"cell_type":"code","source":"train_df = train_df.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:49.881402Z","iopub.execute_input":"2022-05-31T23:43:49.881749Z","iopub.status.idle":"2022-05-31T23:43:50.063691Z","shell.execute_reply.started":"2022-05-31T23:43:49.881720Z","shell.execute_reply":"2022-05-31T23:43:50.062818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:50.064984Z","iopub.execute_input":"2022-05-31T23:43:50.065293Z","iopub.status.idle":"2022-05-31T23:43:50.490268Z","shell.execute_reply.started":"2022-05-31T23:43:50.065267Z","shell.execute_reply":"2022-05-31T23:43:50.489198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_X_cols = train_df.columns[1:-1]","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:50.491746Z","iopub.execute_input":"2022-05-31T23:43:50.492223Z","iopub.status.idle":"2022-05-31T23:43:50.498550Z","shell.execute_reply.started":"2022-05-31T23:43:50.492178Z","shell.execute_reply":"2022-05-31T23:43:50.497854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(\n train_df[_X_cols], train_df[\"target\"], test_size=0.2, \n random_state=42, stratify=train_df[\"target\"],\n)\n\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:50.499894Z","iopub.execute_input":"2022-05-31T23:43:50.500531Z","iopub.status.idle":"2022-05-31T23:43:50.860441Z","shell.execute_reply.started":"2022-05-31T23:43:50.500478Z","shell.execute_reply":"2022-05-31T23:43:50.859289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train random forest model and select the best hyperparameters","metadata":{}},{"cell_type":"code","source":"parameters = {\n \"n_estimators\": [5, 50], \n \"max_depth\": [3, 5],\n}\n\nmodel = RandomForestClassifier(\n random_state=42,\n class_weight=\"balanced\",\n)\n\nmodel = GridSearchCV(\n model, \n parameters, \n cv=5,\n scoring=\"f1\",\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:50.861940Z","iopub.execute_input":"2022-05-31T23:43:50.862371Z","iopub.status.idle":"2022-05-31T23:43:50.868468Z","shell.execute_reply.started":"2022-05-31T23:43:50.862337Z","shell.execute_reply":"2022-05-31T23:43:50.867486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:43:50.870707Z","iopub.execute_input":"2022-05-31T23:43:50.871136Z","iopub.status.idle":"2022-05-31T23:51:23.015618Z","shell.execute_reply.started":"2022-05-31T23:43:50.871101Z","shell.execute_reply":"2022-05-31T23:51:23.014261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:51:23.017907Z","iopub.execute_input":"2022-05-31T23:51:23.018272Z","iopub.status.idle":"2022-05-31T23:51:23.026458Z","shell.execute_reply.started":"2022-05-31T23:51:23.018239Z","shell.execute_reply":"2022-05-31T23:51:23.025424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"See the best features","metadata":{}},{"cell_type":"code","source":"feature_importances = model.best_estimator_.feature_importances_\nvis_indexes = list(range(len(feature_importances)))\nvis_indexes = sorted(vis_indexes, key=lambda x: -feature_importances[x])\nplt.figure(figsize=(8, 8))\nsn.barplot(\n x=feature_importances[vis_indexes], \n y=_X_cols[vis_indexes],\n)\nplt.yticks(fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:42.415430Z","iopub.execute_input":"2022-05-31T23:52:42.415885Z","iopub.status.idle":"2022-05-31T23:52:42.785237Z","shell.execute_reply.started":"2022-05-31T23:52:42.415845Z","shell.execute_reply":"2022-05-31T23:52:42.784002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://www.kaggle.com/code/inversion/amex-competition-metric-python","metadata":{}},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n df = (pd.concat([y_true, y_pred], axis='columns')\n .sort_values('prediction', ascending=False))\n df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n four_pct_cutoff = int(0.04 * df['weight'].sum())\n df['weight_cumsum'] = df['weight'].cumsum()\n df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n \n def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n df = (pd.concat([y_true, y_pred], axis='columns')\n .sort_values('prediction', ascending=False))\n df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n total_pos = (df['target'] * df['weight']).sum()\n df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n df['lorentz'] = df['cum_pos_found'] / total_pos\n df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n return df['gini'].sum()\n\n def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n y_true_pred = y_true.rename(columns={'target': 'prediction'})\n return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n g = normalized_weighted_gini(y_true, y_pred)\n d = top_four_percent_captured(y_true, y_pred)\n\n return 0.5 * (g + d)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-05-31T23:52:42.918781Z","iopub.execute_input":"2022-05-31T23:52:42.919598Z","iopub.status.idle":"2022-05-31T23:52:43.429040Z","shell.execute_reply.started":"2022-05-31T23:52:42.919556Z","shell.execute_reply":"2022-05-31T23:52:43.428213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check train metrics","metadata":{}},{"cell_type":"code","source":"y_pred_train = model.predict_proba(X_train)[:, 1]\nf1_score(y_train, y_pred_train >= 0.5)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:43.430967Z","iopub.execute_input":"2022-05-31T23:52:43.432494Z","iopub.status.idle":"2022-05-31T23:52:45.040976Z","shell.execute_reply.started":"2022-05-31T23:52:43.432456Z","shell.execute_reply":"2022-05-31T23:52:45.039809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nsn.histplot(x=y_pred_train, bins=50);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:45.042750Z","iopub.execute_input":"2022-05-31T23:52:45.043081Z","iopub.status.idle":"2022-05-31T23:52:45.367115Z","shell.execute_reply.started":"2022-05-31T23:52:45.043053Z","shell.execute_reply":"2022-05-31T23:52:45.366166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric(\n pd.DataFrame({\"target\": y_train}).reset_index(drop=True), \n pd.DataFrame({\"prediction\": y_pred_train}).reset_index(drop=True),\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:45.368550Z","iopub.execute_input":"2022-05-31T23:52:45.368878Z","iopub.status.idle":"2022-05-31T23:52:46.202128Z","shell.execute_reply.started":"2022-05-31T23:52:45.368850Z","shell.execute_reply":"2022-05-31T23:52:46.200933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check valid metrics","metadata":{}},{"cell_type":"code","source":"y_pred_valid = model.predict_proba(X_valid)[:, 1]\nf1_score(y_valid, y_pred_valid >= 0.5)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:46.204021Z","iopub.execute_input":"2022-05-31T23:52:46.204365Z","iopub.status.idle":"2022-05-31T23:52:46.618419Z","shell.execute_reply.started":"2022-05-31T23:52:46.204336Z","shell.execute_reply":"2022-05-31T23:52:46.617382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nsn.histplot(x=y_pred_valid, bins=50);","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:46.619880Z","iopub.execute_input":"2022-05-31T23:52:46.620236Z","iopub.status.idle":"2022-05-31T23:52:46.919542Z","shell.execute_reply.started":"2022-05-31T23:52:46.620206Z","shell.execute_reply":"2022-05-31T23:52:46.918306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric(\n pd.DataFrame({\"target\": y_valid}).reset_index(drop=True), \n pd.DataFrame({\"prediction\": y_pred_valid}).reset_index(drop=True),\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:46.921919Z","iopub.execute_input":"2022-05-31T23:52:46.922544Z","iopub.status.idle":"2022-05-31T23:52:47.141266Z","shell.execute_reply.started":"2022-05-31T23:52:46.922475Z","shell.execute_reply":"2022-05-31T23:52:47.140250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DATA_PATH = \"../input/amex-default-prediction/test_data.csv\"\nSAMPLE_SUBMISSION_PATH = \"../input/amex-default-prediction/sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:47.142740Z","iopub.execute_input":"2022-05-31T23:52:47.143870Z","iopub.status.idle":"2022-05-31T23:52:47.148972Z","shell.execute_reply.started":"2022-05-31T23:52:47.143821Z","shell.execute_reply":"2022-05-31T23:52:47.147845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df = pd.read_csv(SAMPLE_SUBMISSION_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:47.150205Z","iopub.execute_input":"2022-05-31T23:52:47.150987Z","iopub.status.idle":"2022-05-31T23:52:49.274200Z","shell.execute_reply.started":"2022-05-31T23:52:47.150944Z","shell.execute_reply":"2022-05-31T23:52:49.273095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunksize = 1000000\n\ntest_df_iter = pd.read_csv(TEST_DATA_PATH, chunksize=chunksize, usecols=[\"customer_ID\"] + X_cols)","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:49.276401Z","iopub.execute_input":"2022-05-31T23:52:49.276808Z","iopub.status.idle":"2022-05-31T23:52:49.288836Z","shell.execute_reply.started":"2022-05-31T23:52:49.276764Z","shell.execute_reply":"2022-05-31T23:52:49.287816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Iterate over chunks of test data and make predictions for them","metadata":{}},{"cell_type":"code","source":"_index = []\n_vals = []\n\nfor chunk in test_df_iter:\n _chunk_mean = chunk.groupby(\"customer_ID\")[X_cols].mean().reset_index()\n _chunk_last = chunk.groupby(\"customer_ID\")[X_cols].last().reset_index()\n _chunk = pd.merge(\n left=_chunk_mean, \n right=_chunk_last, \n how=\"inner\",\n on=\"customer_ID\",\n suffixes=(\"_mean\", \"_last\"),\n )\n\n X_test = _chunk[_X_cols]\n X_test = X_test.fillna(0)\n y_test_pred = model.predict_proba(X_test)[:, 1]\n _index.extend(_chunk[\"customer_ID\"])\n _vals.extend(y_test_pred)\n \n print(len(_index))","metadata":{"execution":{"iopub.status.busy":"2022-05-31T23:52:49.290249Z","iopub.execute_input":"2022-05-31T23:52:49.291065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df = pd.DataFrame(\n {\"customer_ID\": _index, \"prediction\": _vals}\n).groupby(\"customer_ID\")[\"prediction\"].mean().reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"See test distribution of predictions","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nsn.histplot(data=res_df, x=\"prediction\", bins=50);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}