{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# American Express Credict Card Default\n\nPredict if a customer will default in the future [American Express - Default Prediction](https://www.kaggle.com/competitions/amex-default-prediction/overview) challenge \n\n### A. [Preparation](#Preparation)\n* Import Library\n* Define Global Variables and Funtions \n\n### B. [Exploratory Data Analysis](#Overview)\n* [Overview](#Overview)\n* [Visualizations](#Visualizations)\n\n### C. [Feature Engineering](#Data-Processing)\n* [Metrics Selection](#metric-selection)\n* [Explore metrics with Logistic Regression](#Logistic-Regression)\n    0. Train and Evaluate\n    1. Confusion Matrix\n    2. Visualize the Feature Correlation\n\n### D. [Model Training and Evaluation](#Model-Building)\n   **Steps:**\n   \n   0. Get X, y and split train and validation \n   1. Build a Classifier\n   2. Hyperparameter Tuning using RandomizedSearchCV\n   3. Build the final Classifier Model\n   4. Evaluate with Accuracy and Cross Validation \n   5. Visualize the Difference in Actuals and Predictions\n    \n**Classifiers:**\n* [Random Forest](#Random-Forest)\n* [XGBoost](#XGB)\n* [LightGBM](#LGBM)\n\n### E. [Comparison and Selection](#Comparison)\n* Compare and Contrast the performance of Three Classifiers with [Plots](#performance-visualization)\n\n<!--    1. Random Forest\n   2. XGBoost\n   3. LightGBM -->\n   \n### F. [Predict for Test Results](#Test-Results)\n* Get submission.csv for y_test in chunks","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:45:34.674010Z","iopub.execute_input":"2023-09-14T07:45:34.674896Z","iopub.status.idle":"2023-09-14T07:45:34.691045Z","shell.execute_reply.started":"2023-09-14T07:45:34.674846Z","shell.execute_reply":"2023-09-14T07:45:34.688988Z"}}},{"cell_type":"markdown","source":" ","metadata":{}},{"cell_type":"markdown","source":"# Preparation","metadata":{}},{"cell_type":"markdown","source":"import libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import classification_report, confusion_matrix, mean_absolute_error\nfrom sklearn.model_selection import cross_val_score, TimeSeriesSplit\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import f1_score, accuracy_score, ConfusionMatrixDisplay\n\n# import tensorflow_decision_forests as tfdf\n\n# !pip install dexplot\n# import dexplot as dxp\nimport seaborn as sns\nsns.set(style = \"whitegrid\", \n        color_codes = True,\n        font_scale = 1.5)\n\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score\nfrom xgboost.sklearn import XGBClassifier\nimport xgboost as xgb\nimport gc\nimport pickle\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-15T01:34:08.309265Z","iopub.execute_input":"2023-09-15T01:34:08.310865Z","iopub.status.idle":"2023-09-15T01:34:11.271803Z","shell.execute_reply.started":"2023-09-15T01:34:08.310806Z","shell.execute_reply":"2023-09-15T01:34:11.270730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"define global var","metadata":{}},{"cell_type":"code","source":"TRAIN_DATA_PATH = \"../input/amex-default-prediction/train_data.csv\"\nTRAIN_LABELS_PATH = \"../input/amex-default-prediction/train_labels.csv\"\nTEST_DATA_PATH = \"../input/amex-default-prediction/test_data.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:24.994241Z","iopub.execute_input":"2023-09-14T00:00:24.994785Z","iopub.status.idle":"2023-09-14T00:00:25.002788Z","shell.execute_reply.started":"2023-09-14T00:00:24.994742Z","shell.execute_reply":"2023-09-14T00:00:25.001151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"build dataframes and functions for model eval ","metadata":{}},{"cell_type":"code","source":"result_model_df = pd.DataFrame(columns = ['model','train_accuracy','valid_accuracy','train_amex_metric','valid_amex_metric'])\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T01:34:11.273256Z","iopub.execute_input":"2023-09-15T01:34:11.273686Z","iopub.status.idle":"2023-09-15T01:34:11.300842Z","shell.execute_reply.started":"2023-09-15T01:34:11.273649Z","shell.execute_reply":"2023-09-15T01:34:11.299639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Overview","metadata":{}},{"cell_type":"markdown","source":"Dataset is so big, that's why we load first 10000 rows.","metadata":{}},{"cell_type":"code","source":"train_X = pd.read_csv(TRAIN_DATA_PATH, parse_dates=['S_2'], nrows=40000)\ntrain_Y = pd.read_csv(TRAIN_LABELS_PATH, nrows=40000)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:25.981291Z","iopub.execute_input":"2023-09-14T00:00:25.981938Z","iopub.status.idle":"2023-09-14T00:00:29.743333Z","shell.execute_reply.started":"2023-09-14T00:00:25.981887Z","shell.execute_reply":"2023-09-14T00:00:29.742261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's look at one customer","metadata":{}},{"cell_type":"code","source":"example_customer_id = \"000f1c950ae4e388f44e9ba96dd6334dfa85d8be0416d9d0d30228301f2e4cc4\"","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:29.745773Z","iopub.execute_input":"2023-09-14T00:00:29.746275Z","iopub.status.idle":"2023-09-14T00:00:29.751993Z","shell.execute_reply.started":"2023-09-14T00:00:29.746228Z","shell.execute_reply":"2023-09-14T00:00:29.750683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_data_ex = train_X[train_X[\"customer_ID\"] == example_customer_id]\ncustomer_data_ex","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:29.753287Z","iopub.execute_input":"2023-09-14T00:00:29.753706Z","iopub.status.idle":"2023-09-14T00:00:29.824576Z","shell.execute_reply.started":"2023-09-14T00:00:29.753660Z","shell.execute_reply":"2023-09-14T00:00:29.823214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Features are anonymized and normalized and there are 188 variables in total, and fall into the following general categories:\n- D_* = 96 Delinquency variables\n- S_* = 21 Spend variables\n- P_* = 3 Payment variables\n- B_* = 40 Balance variables\n- R_* = 28 Risk variables\n\n\nwith the following features being categorical:\n['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{}},{"cell_type":"code","source":"all_cols = list(customer_data_ex.columns)\nprint(all_cols)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:30.772946Z","iopub.execute_input":"2023-09-14T00:00:30.773577Z","iopub.status.idle":"2023-09-14T00:00:30.781591Z","shell.execute_reply.started":"2023-09-14T00:00:30.773522Z","shell.execute_reply":"2023-09-14T00:00:30.780240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b_cols = list(filter(lambda x: x.startswith(\"B_\"), all_cols))\nprint(b_cols)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:31.554173Z","iopub.execute_input":"2023-09-14T00:00:31.554681Z","iopub.status.idle":"2023-09-14T00:00:31.561313Z","shell.execute_reply.started":"2023-09-14T00:00:31.554627Z","shell.execute_reply":"2023-09-14T00:00:31.560236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check if the customer will future payment default","metadata":{}},{"cell_type":"code","source":"train_Y[train_Y[\"customer_ID\"] == example_customer_id]","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:32.634395Z","iopub.execute_input":"2023-09-14T00:00:32.634855Z","iopub.status.idle":"2023-09-14T00:00:32.653744Z","shell.execute_reply.started":"2023-09-14T00:00:32.634818Z","shell.execute_reply":"2023-09-14T00:00:32.653017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_data_ex.loc[:, \"S_2\"] = pd.to_datetime(customer_data_ex[\"S_2\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:33.332798Z","iopub.execute_input":"2023-09-14T00:00:33.334247Z","iopub.status.idle":"2023-09-14T00:00:33.342469Z","shell.execute_reply.started":"2023-09-14T00:00:33.334193Z","shell.execute_reply":"2023-09-14T00:00:33.341200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsns.lineplot(data=customer_data_ex, x=\"S_2\", y=\"P_2\")\nplt.title(\"P_2\", fontsize=16)\nplt.xlabel(\"date\", fontsize=14)\nplt.ylabel(\"P_2\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:34.096183Z","iopub.execute_input":"2023-09-14T00:00:34.096664Z","iopub.status.idle":"2023-09-14T00:00:34.585853Z","shell.execute_reply.started":"2023-09-14T00:00:34.096606Z","shell.execute_reply":"2023-09-14T00:00:34.584734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizations","metadata":{}},{"cell_type":"markdown","source":"Show only 10 first customer's","metadata":{}},{"cell_type":"code","source":"ex_customer_ids = train_Y.iloc[:10][\"customer_ID\"].tolist()\nex_customer_data = train_X[train_X[\"customer_ID\"].isin(ex_customer_ids)]","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:36.866764Z","iopub.execute_input":"2023-09-14T00:00:36.867527Z","iopub.status.idle":"2023-09-14T00:00:36.882263Z","shell.execute_reply.started":"2023-09-14T00:00:36.867471Z","shell.execute_reply":"2023-09-14T00:00:36.880527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data = pd.merge(ex_customer_data, train_Y.iloc[:10], on=\"customer_ID\")\nex_customer_data[\"S_2\"] = pd.to_datetime(ex_customer_data[\"S_2\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:37.390950Z","iopub.execute_input":"2023-09-14T00:00:37.391460Z","iopub.status.idle":"2023-09-14T00:00:37.409416Z","shell.execute_reply.started":"2023-09-14T00:00:37.391419Z","shell.execute_reply":"2023-09-14T00:00:37.408458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:37.721194Z","iopub.execute_input":"2023-09-14T00:00:37.721853Z","iopub.status.idle":"2023-09-14T00:00:37.754116Z","shell.execute_reply.started":"2023-09-14T00:00:37.721814Z","shell.execute_reply":"2023-09-14T00:00:37.752731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How their feature time series look like","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor _, group in ex_customer_data.groupby(\"customer_ID\"):\n    sns.lineplot(data=group, x=\"S_2\", y=\"P_2\", label=group[\"target\"].max())\nplt.title(\"P_2\", fontsize=16)\nplt.xlabel(\"date\", fontsize=14)\nplt.ylabel(\"P_2\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:39.164260Z","iopub.execute_input":"2023-09-14T00:00:39.164718Z","iopub.status.idle":"2023-09-14T00:00:40.188766Z","shell.execute_reply.started":"2023-09-14T00:00:39.164667Z","shell.execute_reply":"2023-09-14T00:00:40.187616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor _, group in ex_customer_data.groupby(\"customer_ID\"):\n    sns.lineplot(data=group, x=\"S_2\", y=\"B_1\", label=group[\"target\"].max())\nplt.title(\"B_1\", fontsize=16)\nplt.xlabel(\"date\", fontsize=14)\nplt.ylabel(\"B_1\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:41.091003Z","iopub.execute_input":"2023-09-14T00:00:41.091489Z","iopub.status.idle":"2023-09-14T00:00:41.884757Z","shell.execute_reply.started":"2023-09-14T00:00:41.091454Z","shell.execute_reply":"2023-09-14T00:00:41.883499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor _, group in ex_customer_data.groupby(\"customer_ID\"):\n    sns.lineplot(data=group, x=\"S_2\", y=\"B_2\", label=group[\"target\"].max())\nplt.title(\"B_2\", fontsize=16)\nplt.xlabel(\"date\", fontsize=14)\nplt.ylabel(\"B_2\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:00:41.886938Z","iopub.execute_input":"2023-09-14T00:00:41.887379Z","iopub.status.idle":"2023-09-14T00:00:42.698226Z","shell.execute_reply.started":"2023-09-14T00:00:41.887340Z","shell.execute_reply":"2023-09-14T00:00:42.696713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take 1000 customers and show features histograms for target customers and for no-target","metadata":{}},{"cell_type":"code","source":"ex_customer_ids = train_Y.iloc[:1000][\"customer_ID\"].tolist()\nex_customer_data = train_X[train_X[\"customer_ID\"].isin(ex_customer_ids)]\nex_customer_data = pd.merge(ex_customer_data, train_Y.iloc[:1000], on=\"customer_ID\")\nex_customer_data[\"S_2\"] = pd.to_datetime(ex_customer_data[\"S_2\"])\n\nex_customer_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:08.179995Z","iopub.execute_input":"2023-09-14T00:01:08.180507Z","iopub.status.idle":"2023-09-14T00:01:08.267625Z","shell.execute_reply.started":"2023-09-14T00:01:08.180469Z","shell.execute_reply":"2023-09-14T00:01:08.266472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have 735 no-target customers and 265 target","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nsns.countplot(y=ex_customer_data.groupby(\"customer_ID\")[\"target\"].max())\nplt.title(\"Class distribution\", fontsize=16)\nplt.xlabel(\"count\", fontsize=14)\nplt.ylabel(\"target\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:09.572743Z","iopub.execute_input":"2023-09-14T00:01:09.573211Z","iopub.status.idle":"2023-09-14T00:01:09.854228Z","shell.execute_reply.started":"2023-09-14T00:01:09.573177Z","shell.execute_reply":"2023-09-14T00:01:09.853066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 8))\ndf = ex_customer_data.groupby([\"customer_ID\"]).size().to_frame(name='counts').reset_index()\ndf = pd.merge(df, train_Y.iloc[:1000], on=\"customer_ID\", how = 'inner')\nres = df.groupby([\"counts\",\"target\"]).size().to_frame('occurences').reset_index()\nsns.barplot(x= 'counts', y = 'occurences', hue=\"target\", data=res)\nplt.title(\"Distribution of the number of records for the client\", fontsize=16)\nplt.xlabel(\"count\", fontsize=14)\nplt.ylabel(\"n_records\", fontsize=14);\n# px.scatter(ex_customer_data, x=\"date\", y=\"amount_per_item\", color=\"shop_id\",\n#            hover_data=[\"total_items\"], title = \"Order Amount per Item by Shop\")","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:10.752907Z","iopub.execute_input":"2023-09-14T00:01:10.753417Z","iopub.status.idle":"2023-09-14T00:01:11.277545Z","shell.execute_reply.started":"2023-09-14T00:01:10.753376Z","shell.execute_reply":"2023-09-14T00:01:11.276152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 8))\n\nsns.histplot(data=ex_customer_data, x=\"S_2\", bins=100, hue = 'target')\nplt.title(\"Distribution of records by time\", fontsize=16)\nplt.xlabel(\"count\", fontsize=14)\nplt.ylabel(\"n_records\", fontsize=14);","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:11.416492Z","iopub.execute_input":"2023-09-14T00:01:11.417425Z","iopub.status.idle":"2023-09-14T00:01:12.515233Z","shell.execute_reply.started":"2023-09-14T00:01:11.417379Z","shell.execute_reply":"2023-09-14T00:01:12.513986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def sort_f(x):\n    try:\n        a, b = x.split(\"_\")\n        return a, int(b)\n    except:\n        return \"0\", 0\n\nall_cols = sorted(all_cols, key=sort_f)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:12.517257Z","iopub.execute_input":"2023-09-14T00:01:12.517729Z","iopub.status.idle":"2023-09-14T00:01:12.524973Z","shell.execute_reply.started":"2023-09-14T00:01:12.517689Z","shell.execute_reply":"2023-09-14T00:01:12.523697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = [\n    'B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', \n    'D_126', 'D_63', 'D_64', 'D_66', 'D_68',\n]","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:13.573173Z","iopub.execute_input":"2023-09-14T00:01:13.573911Z","iopub.status.idle":"2023-09-14T00:01:13.579492Z","shell.execute_reply.started":"2023-09-14T00:01:13.573871Z","shell.execute_reply":"2023-09-14T00:01:13.578283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind = 0\nfor col in categorical_cols:\n    if ind % 4 == 0:\n        plt.figure(figsize=(16, 3))\n    plt.subplot(1, 4, ind % 4 + 1)\n    \n    sns.countplot(data=ex_customer_data, x=col, hue=\"target\")\n    plt.ylabel(\"\")\n    \n    if ind % 4 == 3:\n        plt.show()\n    ind += 1","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:14.131713Z","iopub.execute_input":"2023-09-14T00:01:14.132229Z","iopub.status.idle":"2023-09-14T00:01:16.595894Z","shell.execute_reply.started":"2023-09-14T00:01:14.132188Z","shell.execute_reply":"2023-09-14T00:01:16.594362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ind = 0\nfor col in all_cols:\n    if col in [\"S_2\", \"customer_ID\", \"target\"] + categorical_cols:\n        continue\n    \n    if ind % 4 == 0:\n        plt.figure(figsize=(16, 4))\n    plt.subplot(1, 4, ind % 4 + 1)\n    \n    sns.histplot(data=ex_customer_data, x=col, hue=\"target\", bins=20)\n    plt.ylabel(\"\")\n    \n    if ind % 4 == 3:\n        plt.show()\n    \n    ind += 1","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:01:16.599046Z","iopub.execute_input":"2023-09-14T00:01:16.599665Z","iopub.status.idle":"2023-09-14T00:02:17.926126Z","shell.execute_reply.started":"2023-09-14T00:01:16.599586Z","shell.execute_reply":"2023-09-14T00:02:17.924868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### show metrics summary","metadata":{}},{"cell_type":"code","source":"ex_customer_data[ex_customer_data[\"target\"] == 0][b_cols[:]].describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:17.928069Z","iopub.execute_input":"2023-09-14T00:02:17.928611Z","iopub.status.idle":"2023-09-14T00:02:18.108184Z","shell.execute_reply.started":"2023-09-14T00:02:17.928573Z","shell.execute_reply":"2023-09-14T00:02:18.107015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data[ex_customer_data[\"target\"] == 1][b_cols[:]].describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:18.110479Z","iopub.execute_input":"2023-09-14T00:02:18.111578Z","iopub.status.idle":"2023-09-14T00:02:18.252028Z","shell.execute_reply.started":"2023-09-14T00:02:18.111535Z","shell.execute_reply":"2023-09-14T00:02:18.251249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_cols = list(filter(lambda x: x.startswith(\"D_\"), all_cols))\nex_customer_data[ex_customer_data[\"target\"] == 0][d_cols[:]].describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:18.253277Z","iopub.execute_input":"2023-09-14T00:02:18.254288Z","iopub.status.idle":"2023-09-14T00:02:18.554051Z","shell.execute_reply.started":"2023-09-14T00:02:18.254245Z","shell.execute_reply":"2023-09-14T00:02:18.553321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data[ex_customer_data[\"target\"] == 1][d_cols[:]].describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:18.555205Z","iopub.execute_input":"2023-09-14T00:02:18.556008Z","iopub.status.idle":"2023-09-14T00:02:18.819405Z","shell.execute_reply.started":"2023-09-14T00:02:18.555970Z","shell.execute_reply":"2023-09-14T00:02:18.818232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r_cols = list(filter(lambda x: x.startswith(\"R_\"), all_cols))\nex_customer_data[ex_customer_data[\"target\"] == 0][r_cols[:]].describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:18.820772Z","iopub.execute_input":"2023-09-14T00:02:18.821116Z","iopub.status.idle":"2023-09-14T00:02:18.941948Z","shell.execute_reply.started":"2023-09-14T00:02:18.821086Z","shell.execute_reply":"2023-09-14T00:02:18.941235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_customer_data[ex_customer_data[\"target\"] == 1][r_cols[:]].describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:18.943129Z","iopub.execute_input":"2023-09-14T00:02:18.943936Z","iopub.status.idle":"2023-09-14T00:02:19.051860Z","shell.execute_reply.started":"2023-09-14T00:02:18.943901Z","shell.execute_reply":"2023-09-14T00:02:19.050979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" ","metadata":{}},{"cell_type":"markdown","source":"# Data Processing","metadata":{}},{"cell_type":"markdown","source":"### Metric Selection","metadata":{}},{"cell_type":"code","source":"X_mean_cols = [\n    \"B_37\", \"B_27\",\n    \"D_42\", \"D_45\", \"D_46\", \"D_47\", \"D_48\", \"D_52\",\n    \"D_54\", \"D_55\", \"D_59\", \"D_61\", \"D_62\", \"D_96\",\n    \"D_105\", \"D_112\", \"D_115\", \"D_118\", \"B_32\",\n    \"D_121\", \"D_122\", \"D_124\", \"D_142\",\n    \"P_2\", \"P_3\", \n    \"S_3\", \"S_6\", \"S_7\", \"S_11\", \"S_15\", \"S_16\", \"S_17\",\"S_19\",\n    \"S_20\", \"S_22\",\"S_23\", \"S_26\", \"S_27\",\n    \"R_1\", \"R_3\", \"R_13\", \"R_18\", \"R_27\", \"R_28\"\n]\nX_last_cols = [\n    \"B_2\", \"B_3\", \"B_4\", \"B_5\", \"B_7\", \"B_9\",\n    \"B_18\", \"B_20\", \"B_23\", \"B_39\",\n    \"D_87\", \"D_88\",\"D_110\", \"D_111\", \"D_119\", \n    \"D_134\", \"D_135\", \"D_136\", \"D_137\", \"D_138\",\n]\ncategorical_cols = [\n    'B_30', 'B_38',\n    'D_114', 'D_116', 'D_117', 'D_120', \n    'D_126', 'D_63', 'D_64', 'D_66', 'D_68',\n]","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:19.054528Z","iopub.execute_input":"2023-09-14T00:02:19.054930Z","iopub.status.idle":"2023-09-14T00:02:19.064833Z","shell.execute_reply.started":"2023-09-14T00:02:19.054896Z","shell.execute_reply":"2023-09-14T00:02:19.063692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"def process_cat(df, col):\n    one_hot = pd.get_dummies(df[col], prefix = col)\n    df = df.drop(col, axis = 1).join(one_hot)\n    return df\ndef get_time(df):\n    df['Date'] = pd.to_datetime(df['S_2'])\n    df['month'] = pd.DatetimeIndex(df['Date']).month\n    df['weekofyear'] = pd.DatetimeIndex(df['Date']).weekofyear\n    df['weekday'] = pd.DatetimeIndex(df['Date']).weekday\n    return df\ndef getX(df, _X_cols = []):\n    df = get_time(df)\n    _chunk_mean = df.groupby(\"customer_ID\")[X_mean_cols].mean().reset_index()\n    _chunk_last = df.groupby(\"customer_ID\")[X_last_cols].last().reset_index()\n    _chunk = pd.merge(\n        left=_chunk_mean, \n        right=_chunk_last, \n        how=\"inner\",\n        on=\"customer_ID\",\n        suffixes=(\"_mean\", \"_last\"),\n    )\n    _chunk = _chunk.fillna(method='ffill').fillna(method='bfill')\n    \n    _chunk_dt = df.loc[:, [\"customer_ID\", 'weekday', 'month']]\n    _chunk_dt = process_cat(_chunk_dt, 'weekday')\n    _chunk_dt = process_cat(_chunk_dt, 'month')\n    _chunk = pd.merge(_chunk, _chunk_dt, on=\"customer_ID\", how=\"inner\")\n    \n    _chunk_cat = df.loc[:, [\"customer_ID\"] + categorical_cols]\n    _chunk_cat = process_cat(_chunk_cat, 'D_64')\n    _chunk_cat = process_cat(_chunk_cat, 'D_63')\n    _chunk = pd.merge(_chunk, _chunk_cat, on=\"customer_ID\", how=\"inner\")\n    \n    if (_X_cols != []):\n        missing_cols = set(_X_cols) - set( _chunk.columns )\n        for c in missing_cols:\n            _chunk[c] = 0\n        _chunk = _chunk[_X_cols]\n    _chunk = _chunk.fillna(0)\n    return _chunk    ","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:19.066403Z","iopub.execute_input":"2023-09-14T00:02:19.066845Z","iopub.status.idle":"2023-09-14T00:02:19.085879Z","shell.execute_reply.started":"2023-09-14T00:02:19.066812Z","shell.execute_reply":"2023-09-14T00:02:19.084478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logistic Regression","metadata":{}},{"cell_type":"markdown","source":"### In-Sample Evaluation","metadata":{}},{"cell_type":"code","source":"train_df = getX(train_X)\ntrain_df = pd.merge(train_df, train_Y, on=\"customer_ID\", how=\"left\")\n\nX_train = train_df.iloc[:, 1:-1]\nY_train = train_df.iloc[:, -1]\nmodel = LogisticRegression(solver = 'lbfgs')\nmodel.fit(X_train, Y_train)\n\npredictions = model.predict(X_train)\nvalid_predictions= model.predict(X_train)\n\nprint(f\"Training F1 Score: {np.mean(f1_score(Y_train, predictions)).round(4)}\")\nprint(f\"Training Accuracy: {np.mean(accuracy_score(Y_train, predictions)).round(4)}\")\nprint(cross_val_score(model, X_train, Y_train, cv = TimeSeriesSplit()))","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:02:19.087555Z","iopub.execute_input":"2023-09-14T00:02:19.088011Z","iopub.status.idle":"2023-09-14T00:03:30.888341Z","shell.execute_reply.started":"2023-09-14T00:02:19.087972Z","shell.execute_reply":"2023-09-14T00:03:30.886916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion Matrix","metadata":{}},{"cell_type":"code","source":"disp = ConfusionMatrixDisplay.from_estimator(model, X_train, Y_train,\n                                             display_labels=['Absent', 'Present'],\n                                             cmap='Blues') \ndisp.figure_.set_size_inches((7, 7))\ndisp.ax_.set_title('Model Level 1: Logistic\\nRegression Model In- Sample Results')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:03:30.890457Z","iopub.execute_input":"2023-09-14T00:03:30.891376Z","iopub.status.idle":"2023-09-14T00:03:31.513947Z","shell.execute_reply.started":"2023-09-14T00:03:30.891307Z","shell.execute_reply":"2023-09-14T00:03:31.512698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Corr Map","metadata":{}},{"cell_type":"code","source":"plt.rcParams[\"figure.figsize\"] = (80,80)\ncorr = train_df.corr()\nmatrix = np.triu(corr)\nsns.heatmap(corr, center = 0, annot=True, fmt='.1g', cmap=\"YlGnBu\", mask = matrix)\nplt.title(\"Correlation of the Metrics\")","metadata":{"execution":{"iopub.status.busy":"2023-09-14T00:03:31.515390Z","iopub.execute_input":"2023-09-14T00:03:31.515794Z","iopub.status.idle":"2023-09-14T00:04:13.515517Z","shell.execute_reply.started":"2023-09-14T00:03:31.515758Z","shell.execute_reply":"2023-09-14T00:04:13.513441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!--  -->","metadata":{}},{"cell_type":"markdown","source":"# Model Building ","metadata":{}},{"cell_type":"markdown","source":"#### get X, y and split","metadata":{}},{"cell_type":"code","source":"_X_cols = train_df.columns[1:-1]\nprint(_X_cols)\n\nX_train, X_valid, y_train, y_valid = train_test_split(\n    train_df[_X_cols], train_df[\"target\"], test_size=0.3, \n    random_state = 42, stratify=train_df[\"target\"],\n)\n\nX_train.shape, X_valid.shape, y_train.shape, y_valid.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:15:53.573033Z","iopub.execute_input":"2023-09-14T06:15:53.573549Z","iopub.status.idle":"2023-09-14T06:15:55.530729Z","shell.execute_reply.started":"2023-09-14T06:15:53.573505Z","shell.execute_reply":"2023-09-14T06:15:55.529339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- # model-jump -->","metadata":{}},{"cell_type":"markdown","source":"#### build model and Hyperparameter Tuning using RandomizedSearchCV, return the best model","metadata":{}},{"cell_type":"code","source":"def bestModel(classifier, parameters, cv = 5, verbose = 3, n_jobs=-1, random_state=42):\n    classifier = classifier\n    parameters = parameters\n    CV = RandomizedSearchCV(classifier, param_distributions = parameters, \n                            cv = cv, verbose = verbose, n_jobs= n_jobs, random_state=random_state)\n    CV.fit(X_train, y_train)\n    return CV","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:44:53.774757Z","iopub.execute_input":"2023-09-14T06:44:53.775364Z","iopub.status.idle":"2023-09-14T06:48:48.191948Z","shell.execute_reply.started":"2023-09-14T06:44:53.775315Z","shell.execute_reply":"2023-09-14T06:48:48.190697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### model evaluation with accuracy and compare across different models","metadata":{}},{"cell_type":"code","source":"def evalModel(model_name, best_model, cv = 2):\n    print('Result for ', model_name)\n    score = cross_val_score(best_model, X_train, y_train, cv = cv)\n    print('Cross Validate Score: ', score)\n    \n    if model_name == 'random forest':\n        y_pred_train = best_model.predict(X_train)\n        print(\"Accuracy for train data: \", accuracy_score(y_train, y_pred_train))\n        y_pred_valid = best_model.predict(X_valid)\n        print(\"Accuracy for validation data: \", accuracy_score(y_valid, y_pred_valid))\n    \n#         y_pred_train = best_model.predict_proba(X_train)[:, 1]\n#         y_pred_valid = best_model.predict_proba(X_valid)[:, 1]\n#         print(\"f1_score for train:\", f1_score(y_train, y_pred_train >= 0.5))\n#         print(\"f1_score for valid:\", f1_score(y_valid, y_pred_valid >= 0.5))\n        \n    else:\n        y_pred_train = best_model.predict(X_train)\n        y_pred_valid = best_model.predict(X_valid)\n        print(\"Accuracy for train data: \", accuracy_score(y_train, y_pred_train))\n        print(\"Accuracy for validation data: \", accuracy_score(y_valid, y_pred_valid))\n        \n#         print('Precision: %.3f' % f1_score(y_test, y_pred))\n#         print('Precision: %.3f' % f1_score(y_test, y_pred))\n\n\n    train_amex_metric = amex_metric(\n        pd.DataFrame({\"target\": y_train}).reset_index(drop=True), \n        pd.DataFrame({\"prediction\": y_pred_train}).reset_index(drop=True),\n    )\n    valid_amex_metric = amex_metric(\n        pd.DataFrame({\"target\": y_valid}).reset_index(drop=True), \n        pd.DataFrame({\"prediction\": y_pred_valid}).reset_index(drop=True),\n    )\n    print('amex_metric ', 'for train', train_amex_metric)\n    print('amex_metric ', 'for valid', valid_amex_metric)\n    \n#     'random forest'\n    r_list = [model_name, accuracy_score(y_train, y_pred_train), accuracy_score(y_valid, y_pred_valid),\n              train_amex_metric,valid_amex_metric]\n    result_model_df.loc[len(result_model_df)] = r_list\n    \n\n    fig, axs = plt.subplots(1, 2, figsize=(18, 6))\n    fig.suptitle(''+'y_pred train vs valid')\n    axs[0].hist(y_pred_train, bins=50)\n    axs[1].hist(y_pred_valid, bins=50)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T01:42:00.244114Z","iopub.execute_input":"2023-09-15T01:42:00.244673Z","iopub.status.idle":"2023-09-15T01:42:00.262685Z","shell.execute_reply.started":"2023-09-15T01:42:00.244630Z","shell.execute_reply":"2023-09-15T01:42:00.261362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random Forest","metadata":{}},{"cell_type":"code","source":"rfc = RandomForestClassifier()\n\nparams = {\n    \"n_estimators\": [50, 100, 150, 200, 250], \n    \"max_depth\": [3, 5, 8, 10, 12],\n#     'min_sample_leaf':[1, 50, 100, 200, 400, 500],\n#     'max_terminal_nodes':[0, 5, 10, 15, 20, 25],\n    'class_weight':['balanced', 'balanced_subsample'],\n    'max_features': ['auto', 'sqrt', 'log2'],\n    'criterion': ['gini','entropy','log_loss'],\n    'random_state':[42]\n}\n\nCV_rfc = bestModel(rfc, params)\nprint(CV_rfc.best_params_)\nbest_rfc = CV_rfc.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2023-09-14T08:43:20.156408Z","iopub.execute_input":"2023-09-14T08:43:20.157001Z","iopub.status.idle":"2023-09-14T08:43:20.196501Z","shell.execute_reply.started":"2023-09-14T08:43:20.156960Z","shell.execute_reply":"2023-09-14T08:43:20.194464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evalModel('random forest', best_rfc)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rfc = RandomForestClassifier(**CV_rfc.best_estimator_)\n# rfc0 = time.time()\n# rfc.fit(X_train, y_train)\n# rfc1 = time.time()-rfc0\n# # print(\"Training time:\", rfc1)\n\n# pred_train = rfc.predict(X_train)\n# print(\"Accuracy for Random Forest on train data: \", accuracy_score(y_train, pred_train))\n# pred_valid = rfc.predict(X_valid)\n# print(\"Accuracy for Random Forest on validation data: \", accuracy_score(y_valid, pred_valid))","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:03:25.783809Z","iopub.status.idle":"2023-09-14T06:03:25.784334Z","shell.execute_reply.started":"2023-09-14T06:03:25.784096Z","shell.execute_reply":"2023-09-14T06:03:25.784122Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred_train = rfc.predict_proba(X_train)[:, 1]\n# y_pred_valid = rfc.predict_proba(X_valid)[:, 1]\n# print(\"f1_score for train:\", f1_score(y_train, y_pred_train >= 0.5))\n# print(\"f1_score for valid:\", f1_score(y_train, y_pred_valid >= 0.5))","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rfc_train_amex_metric = amex_metric(\n#     pd.DataFrame({\"target\": y_train}).reset_index(drop=True), \n#     pd.DataFrame({\"prediction\": y_pred_train}).reset_index(drop=True),\n# )\n# rfc_valid_amex_metric = amex_metric(\n#     pd.DataFrame({\"target\": y_valid}).reset_index(drop=True), \n#     pd.DataFrame({\"prediction\": y_pred_valid}).reset_index(drop=True),\n# )\n# print('amex_metric', 'for train', rfc_train_amex_metric)\n# print('amex_metric', 'for valid', rfc_valid_amex_metric)\n\n# rfc_list = ['random forest', accuracy_score(y_train, pred_train), accuracy_score(y_valid, pred_valid),\n#             rfc_train_amex_metric,rfc_valid_amex_metric, rfc1]\n# result_model_df.loc[len(result_model_df)] = rfc_list\n\n# fig, axs = plt.subplots(1, 2, figsize=(18, 6))\n# fig.suptitle('y_pred train vs valid')\n# axs[0].hist(y_pred_train, bins=50)\n# axs[1].hist(y_pred_valid, bins=50)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T02:30:54.021032Z","iopub.execute_input":"2023-09-14T02:30:54.021591Z","iopub.status.idle":"2023-09-14T02:31:08.143285Z","shell.execute_reply.started":"2023-09-14T02:30:54.021552Z","shell.execute_reply":"2023-09-14T02:31:08.141982Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"_kg_hide-input":true,"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGB","metadata":{}},{"cell_type":"code","source":"xgb = XGBClassifier()\nparams = {\n    'n_estimators' : [50, 100, 150, 200, 250], \n    'learning_rate' : [0.01, 0.1, 0.25, 0.5, 1],\n    'max_depth' : [ 3, 4, 5, 6, 8, 10, 12, 15],\n    'reg_lambda': [0.1, 1.0, 5.0, 10.0, 50.0],\n    'min_child_samples':[20, 200, 1000, 2000, 2500],\n    'num_leaves':[31, 61, 91, 121],\n    'min_child_weight': [0.5, 1.0, 3.0, 5.0, 7.0], \n    'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],\n    'gamma': [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],\n    'colsample_bytree' : [ 0.3, 0.4, 0.5 , 0.7 ],\n    'random_state' : [42]\n}\nCV_xgb = bestModel(xgb, params)\nprint(CV_xgb.best_params_)\nbest_xgb = CV_xgb.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:08:00.760814Z","iopub.execute_input":"2023-09-14T06:08:00.761368Z","iopub.status.idle":"2023-09-14T06:08:58.116338Z","shell.execute_reply.started":"2023-09-14T06:08:00.761325Z","shell.execute_reply":"2023-09-14T06:08:58.114696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evalModel('xgboost', best_xgb)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgb = XGBClassifier(**CV_xgb.best_params_)\n# xgb0 = time.time()\n# xgb.fit(X_train, y_train, **opt_params)\n# xg1 = time.time()-xgb0\n\n# y_pred_train = xgb.predict(X_train, num_iteration = xgb.best_iteration_)\n# y_pred_valid = xgb.predict(X_valid, num_iteration = xgb.best_iteration_)\n# print(\"Accuracy for xgboost on train data: \", accuracy_score(y_train, y_pred_train))\n# print(\"Accuracy for xgboost on validation data: \", accuracy_score(y_valid, y_pred_valid))","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:07:37.245032Z","iopub.status.idle":"2023-09-14T06:07:37.245809Z","shell.execute_reply.started":"2023-09-14T06:07:37.245423Z","shell.execute_reply":"2023-09-14T06:07:37.245456Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_amex_metric = amex_metric(\n#     pd.DataFrame({\"target\": y_train}).reset_index(drop=True), \n#     pd.DataFrame({\"prediction\": y_pred_train}).reset_index(drop=True),\n# )\n# valid_amex_metric = amex_metric(\n#     pd.DataFrame({\"target\": y_valid}).reset_index(drop=True), \n#     pd.DataFrame({\"prediction\": y_pred_valid}).reset_index(drop=True),\n# )\n# print('amex_metric', 'for train', train_amex_metric)\n# print('amex_metric', 'for valid', valid_amex_metric)\n\n# xgb_list = ['xgboost', accuracy_score(y_train, y_pred_train), accuracy_score(y_valid, y_pred_valid),\n#             train_amex_metric,valid_amex_metric, xgb1]\n# result_model_df.loc[len(result_model_df)] = xgb_list\n\n# fig, axs = plt.subplots(1, 2, figsize=(18, 6))\n# fig.suptitle('y_pred train vs valid')\n# axs[0].hist(y_pred_train, bins=50)\n# axs[1].hist(y_pred_valid, bins=50)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:00:33.014181Z","iopub.status.idle":"2023-09-14T06:00:33.014959Z","shell.execute_reply.started":"2023-09-14T06:00:33.014711Z","shell.execute_reply":"2023-09-14T06:00:33.014735Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM","metadata":{}},{"cell_type":"code","source":"gbm = LGBMClassifier()\nparams = {\n    'n_estimators' : [50, 100, 150, 200, 250], \n    'learning_rate' : [0.01, 0.1, 0.25, 0.5, 1],\n    'reg_lambda': [0.1, 1.0, 5.0, 10.0, 50.0],\n    'min_child_samples':[20, 200, 1000, 2000, 2500],\n    'num_leaves':[31, 61, 91, 121],\n    'min_child_weight': [0.5, 1.0, 3.0, 5.0, 7.0], \n    'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],\n    'max_bins':[100, 200, 500, 750, 1000],\n    'random_state' : [42]\n}\nCV_gbm = bestModel(gbm, params)\nprint(CV_gbm.best_params_)\nbest_gbm = CV_gbm.best_estimator_","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evalModel('lightgbm', best_gbm)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:53:54.617326Z","iopub.execute_input":"2023-09-14T07:53:54.618088Z","iopub.status.idle":"2023-09-14T07:53:54.655390Z","shell.execute_reply.started":"2023-09-14T07:53:54.618040Z","shell.execute_reply":"2023-09-14T07:53:54.654111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gbm = LGBMClassifier()\n# params = {\n#     'n_estimators' : [50, 100, 150, 200, 250], \n#     'learning_rate' : [0.01, 0.1, 0.25, 0.5, 1],\n#     'reg_lambda': [0.1, 1.0, 5.0, 10.0, 50.0, 100.0],\n#     'min_child_samples':[20, 200, 1000, 2000, 2500],\n#     'num_leaves':[31, 61, 91, 121],\n#     'min_child_weight': [0.5, 1.0, 3.0, 5.0, 7.0, 10.0], \n#     'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],\n#     'max_bins':[100, 200, 500, 750, 1000],\n#     'random_state' : [42]\n# }\n# CV_gbm = RandomizedSearchCV(gbm, params, verbose = 3, cv = 2, n_jobs=-1, random_state=42)\n# CV_gbm.fit(X_valid, y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:01:53.340946Z","iopub.execute_input":"2023-09-14T07:01:53.341515Z","iopub.status.idle":"2023-09-14T07:01:53.348090Z","shell.execute_reply.started":"2023-09-14T07:01:53.341471Z","shell.execute_reply":"2023-09-14T07:01:53.346599Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gbm = LGBMClassifier(random_state=42, subsample = 0.7)\n\n# params = {\n#     'n_estimators' : [100, 250], \n#     'learning_rate' : [0.25, 1],\n#     'reg_lambda': [5.0, 50.0],\n#     'min_child_samples':[200, 2500],\n#     'num_leaves':[121, 200],\n#     'min_child_weight': [0.5, 1.0], \n#     'max_bins':[100, 500]\n# }\n# CV_gbm = RandomizedSearchCV(gbm, params, cv = 2, verbose = 3)\n# CV_gbm.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T05:28:10.687612Z","iopub.execute_input":"2023-09-14T05:28:10.688074Z","iopub.status.idle":"2023-09-14T05:28:10.693777Z","shell.execute_reply.started":"2023-09-14T05:28:10.688030Z","shell.execute_reply":"2023-09-14T05:28:10.692584Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_gbm = CV_gbm.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:18:20.729252Z","iopub.execute_input":"2023-09-14T06:18:20.729863Z","iopub.status.idle":"2023-09-14T06:18:20.735922Z","shell.execute_reply.started":"2023-09-14T06:18:20.729815Z","shell.execute_reply":"2023-09-14T06:18:20.734797Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# score=cross_val_score(best_gbm, X_valid, y_valid, cv=10)\n# print(score)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:01:42.188141Z","iopub.execute_input":"2023-09-14T07:01:42.188620Z","iopub.status.idle":"2023-09-14T07:01:42.194850Z","shell.execute_reply.started":"2023-09-14T07:01:42.188582Z","shell.execute_reply":"2023-09-14T07:01:42.193353Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gbm = LGBMClassifier(**CV_gbm.best_params_)\n# #     n_estimators=100,\n# #                      learning_rate=1, \n# #                      reg_lambda=50,\n# #                      min_child_samples=2400,\n# #                      num_leaves=95,\n# #                      min_child_weight=0.5,\n# #                      max_bins=511, \n# #                      random_state=42)\n\n# gbm0 = time.time()\n# gbm.fit(X_valid, y_valid)\n# gbm1 = time.time()-gbm0","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gbm = LGBMClassifier(**CV_gbm.best_params_)\n# #     n_estimators=100,\n# #                      learning_rate=1, \n# #                      reg_lambda=50,\n# #                      min_child_samples=2400,\n# #                      num_leaves=95,\n# #                      min_child_weight=0.5,\n# #                      max_bins=511, \n# #                      random_state=42)\n\n# gbm0 = time.time()\n# gbm.fit(X_train, y_train)\n# gbm1 = time.time()-gbm0","metadata":{"execution":{"iopub.status.busy":"2023-09-14T06:00:32.999013Z","iopub.status.idle":"2023-09-14T06:00:32.999489Z","shell.execute_reply.started":"2023-09-14T06:00:32.999275Z","shell.execute_reply":"2023-09-14T06:00:32.999297Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred_train = best_gbm.predict(X_train, num_iteration = gbm.best_iteration_)\n# y_pred_valid = best_gbm.predict(X_valid, num_iteration = gbm.best_iteration_)\n# print(\"Accuracy for lightgbm on train data: \", accuracy_score(y_train, y_pred_train))\n# print(\"Accuracy for lightgbm on validation data: \", accuracy_score(y_valid, y_pred_valid))","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:02:02.202826Z","iopub.execute_input":"2023-09-14T07:02:02.203386Z","iopub.status.idle":"2023-09-14T07:02:02.210361Z","shell.execute_reply.started":"2023-09-14T07:02:02.203345Z","shell.execute_reply":"2023-09-14T07:02:02.208709Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gbm_train_amex_metric = amex_metric(\n#     pd.DataFrame({\"target\": y_train}).reset_index(drop=True), \n#     pd.DataFrame({\"prediction\": y_pred_train}).reset_index(drop=True),\n# )\n# gbm_valid_amex_metric = amex_metric(\n#     pd.DataFrame({\"target\": y_valid}).reset_index(drop=True), \n#     pd.DataFrame({\"prediction\": y_pred_valid}).reset_index(drop=True),\n# )\n# print('amex_metric', 'for train', gbm_train_amex_metric)\n# print('amex_metric', 'for valid', gbm_valid_amex_metric)\n\n# gbm_list = ['lightgbm', accuracy_score(y_train, y_pred_train), accuracy_score(y_valid, y_pred_valid),\n#             rfc_train_amex_metric,rfc_valid_amex_metric, gbm1]\n# result_model_df.loc[len(result_model_df)] = gbm_list\n\n# fig, axs = plt.subplots(1, 2, figsize=(18, 6))\n# fig.suptitle('y_pred train vs valid')\n# axs[0].hist(y_pred_train, bins=50)\n# axs[1].hist(y_pred_valid, bins=50)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:02:03.064002Z","iopub.execute_input":"2023-09-14T07:02:03.064541Z","iopub.status.idle":"2023-09-14T07:02:03.071039Z","shell.execute_reply.started":"2023-09-14T07:02:03.064501Z","shell.execute_reply":"2023-09-14T07:02:03.069688Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comparison","metadata":{}},{"cell_type":"markdown","source":"#### view the performance_df","metadata":{}},{"cell_type":"code","source":"result_model_df","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:02:10.122915Z","iopub.execute_input":"2023-09-14T07:02:10.123482Z","iopub.status.idle":"2023-09-14T07:02:10.140232Z","shell.execute_reply.started":"2023-09-14T07:02:10.123444Z","shell.execute_reply":"2023-09-14T07:02:10.138831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# saved_model = '/kaggle/input/eval-models-df-saved-from-v12/saved.csv'\n# result_model_df = pd.read_csv(saved_model)\n# result_model_df","metadata":{"execution":{"iopub.status.busy":"2023-09-15T01:34:26.113448Z","iopub.execute_input":"2023-09-15T01:34:26.113931Z","iopub.status.idle":"2023-09-15T01:34:26.144474Z","shell.execute_reply.started":"2023-09-15T01:34:26.113894Z","shell.execute_reply":"2023-09-15T01:34:26.143696Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_model_df.to_csv(\"save_model_result.csv\", index=False)\n# res_df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### performance visualization","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12, 8))        \nresult_model_df.plot.bar(ax=ax, x='model', title='performance by model')   \n# x='model',\n#         kind='bar',\n#         stacked=False,\n#         )\nplt.ylim([0.988, 1.001])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T01:38:50.013323Z","iopub.execute_input":"2023-09-15T01:38:50.014642Z","iopub.status.idle":"2023-09-15T01:38:50.338691Z","shell.execute_reply.started":"2023-09-15T01:38:50.014529Z","shell.execute_reply":"2023-09-15T01:38:50.337480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axs = plt.subplots(1, 2,figsize=(40,10))\n# e_range = [['train_accuracy','valid_accuracy'],['train_amex_metric','valid_amex_metric']]\n# range_p = 2\n\n# for i in range(range_p):\n#     axs[k].bar(x + offset, measurement, width, label=attribute)","metadata":{"jupyter":{"source_hidden":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axs = plt.subplots(1, 2,figsize=(40,10))\n# md = result_model_df['model'].values\n\n# e_range = [['train_accuracy','valid_accuracy'],['train_amex_metric','valid_amex_metric']]\n# k = 0\n# for i in e_range:\n#     x = np.arange(len(i))  # the label locations\n#     width = 0.1  # the width of the bars\n#     multiplier = 0\n#     means = result_model_df[i]\n    \n#     for attribute, measurement in means.items():\n#         print(attribute, measurement)\n#         offset = width * multiplier\n#         rects = axs[k].bar(x + offset, measurement, width, label=attribute)\n#         axs[k].bar_label(rects, padding=5)\n#         multiplier += 1\n    \n#     axs[k].set_ylabel(i)\n#     axs[k].set_title(i[0][6:] + ' by models')\n# #     axs[k].set_xticks(x * width)\n#     axs[k].legend(loc = 'upper right')\n#     k+=1\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T21:10:04.175192Z","iopub.execute_input":"2023-09-14T21:10:04.175800Z","iopub.status.idle":"2023-09-14T21:10:04.599711Z","shell.execute_reply.started":"2023-09-14T21:10:04.175753Z","shell.execute_reply":"2023-09-14T21:10:04.598378Z"},"jupyter":{"source_hidden":true},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(result_model_df.columns)\n# print([result_model_df[i].values for i in result_c])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-15T01:37:49.327879Z","iopub.execute_input":"2023-09-15T01:37:49.328324Z","iopub.status.idle":"2023-09-15T01:37:49.333456Z","shell.execute_reply.started":"2023-09-15T01:37:49.328290Z","shell.execute_reply":"2023-09-15T01:37:49.332159Z"},"_kg_hide-output":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #define points values by group\n# # A = result_model_df.loc[result_model_df['model'] == 'lightgbm', ['train_accuracy','valid_accuracy']]\n# # B = result_model_df.loc[result_model_df['model'] == 'lightgbm', ['train_amex_metric','valid_amex_metric']]\n# # B = result_model_df.loc[result_model_df['model'] == 'random forest', ['train_accuracy','valid_accuracy']]\n# result_c = ['train_accuracy', 'valid_accuracy', 'train_amex_metric','valid_amex_metric', 'train_time']\n# #add three histograms to one plot\n# plt.bar(result_c, [result_model_df[i] for i in result_c],alpha=0.5, label=result_model_df['model'])\n# # plt.hist(B, alpha=0.5, label='random forest')\n\n# #add plot title and axis labels\n# plt.title('Points Distribution by Team')\n# plt.xlabel('Points')\n# plt.ylabel('Frequency')\n\n# #add legend\n# plt.legend(title='Team')\n\n# #display plot\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-14T05:48:46.992801Z","iopub.execute_input":"2023-09-14T05:48:46.993533Z","iopub.status.idle":"2023-09-14T05:48:46.999475Z","shell.execute_reply.started":"2023-09-14T05:48:46.993472Z","shell.execute_reply":"2023-09-14T05:48:46.998520Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result_model_df.hist(['train_accuracy', 'valid_accuracy'],by='model', legend=True, figsize=(40,20))","metadata":{"execution":{"iopub.status.busy":"2023-09-14T04:56:11.375923Z","iopub.execute_input":"2023-09-14T04:56:11.376869Z","iopub.status.idle":"2023-09-14T04:56:11.380922Z","shell.execute_reply.started":"2023-09-14T04:56:11.376825Z","shell.execute_reply":"2023-09-14T04:56:11.380213Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.read_csv(TEST_DATA_PATH,\n#                       usecols=[\"customer_ID\"] + ['S_2'] + X_mean_cols + X_last_cols + categorical_cols)\n\n# feature_importances = model.best_estimator_.feature_importances_\n# vis_indexes = list(range(len(feature_importances)))\n# vis_indexes = sorted(vis_indexes, key=lambda x: -feature_importances[x])\n# plt.figure(figsize=(20, 10))\n# sn.barplot(\n#     x=feature_importances[vis_indexes], \n#     y=_X_cols[vis_indexes],\n# )\n# plt.yticks(fontsize=14);\n# using the feature importance variable\n# feature_imp = pd.Series(rfc.best_estimator_.feature_importances_, \n#                         index = _X_cols).sort_values(ascending = False)\n# feature_imp.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-14T01:58:42.932873Z","iopub.execute_input":"2023-09-14T01:58:42.934331Z","iopub.status.idle":"2023-09-14T01:58:42.940163Z","shell.execute_reply.started":"2023-09-14T01:58:42.934271Z","shell.execute_reply":"2023-09-14T01:58:42.938737Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Results","metadata":{}},{"cell_type":"markdown","source":"choose lightgbm since it has the highest accuracy and highest amex_metric","metadata":{}},{"cell_type":"code","source":"model = best_gbm","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:10:04.766327Z","iopub.execute_input":"2023-09-14T07:10:04.766827Z","iopub.status.idle":"2023-09-14T07:10:04.772749Z","shell.execute_reply.started":"2023-09-14T07:10:04.766789Z","shell.execute_reply":"2023-09-14T07:10:04.771419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Iterate over chunks of test data and make predictions for them","metadata":{}},{"cell_type":"code","source":"chunksize = 100000\ntest_df = pd.read_csv(TEST_DATA_PATH, \n                      chunksize=chunksize, \n                      parse_dates=['S_2'],\n                      usecols=[\"customer_ID\"] + ['S_2'] + X_mean_cols + X_last_cols + categorical_cols)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:10:12.208353Z","iopub.execute_input":"2023-09-14T07:10:12.208905Z","iopub.status.idle":"2023-09-14T07:10:12.229149Z","shell.execute_reply.started":"2023-09-14T07:10:12.208861Z","shell.execute_reply":"2023-09-14T07:10:12.228252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_index = []\n_vals = []\n\nfor chunk in test_df:\n    chunk['month'] = pd.DatetimeIndex(chunk['S_2']).month\n    chunk['weekday'] = pd.DatetimeIndex(chunk['S_2']).weekday\n\n    _chunk_mean = chunk.groupby(\"customer_ID\")[X_mean_cols].mean().reset_index()\n    _chunk_last = chunk.groupby(\"customer_ID\")[X_last_cols].last().reset_index()\n    _chunk = pd.merge(\n        left=_chunk_mean, \n        right=_chunk_last, \n        how=\"inner\",\n        on=\"customer_ID\",\n        suffixes=(\"_mean\", \"_last\"),\n    )\n    _chunk = _chunk.fillna(method='ffill').fillna(method='bfill')\n    \n    _chunk_dt = chunk.loc[:, [\"customer_ID\", 'weekday', 'month']]\n    _chunk_dt = process_cat(_chunk_dt, 'weekday')\n    _chunk_dt = process_cat(_chunk_dt, 'month')\n    _chunk = pd.merge(_chunk, _chunk_dt, on=\"customer_ID\", how=\"inner\")\n    \n    _chunk_cat = chunk.loc[:, [\"customer_ID\"] + categorical_cols]\n    _chunk_cat = process_cat(_chunk_cat, 'D_64')\n    _chunk_cat = process_cat(_chunk_cat, 'D_63')\n    _chunk = pd.merge(_chunk, _chunk_cat, on=\"customer_ID\", how=\"inner\")\n    \n    missing_cols = set(_X_cols) - set(_chunk.columns)\n    for c in missing_cols:\n        _chunk[c] = 0\n    \n    X_test = _chunk[_X_cols]\n    X_test = X_test.fillna(0)\n    y_test_pred = model.predict_proba(X_test)[:, 1]\n    _index.extend(_chunk[\"customer_ID\"])\n    _vals.extend(y_test_pred)\n    print(len(_index))","metadata":{"execution":{"iopub.status.busy":"2023-09-14T07:10:15.564708Z","iopub.execute_input":"2023-09-14T07:10:15.565754Z","iopub.status.idle":"2023-09-14T07:11:08.996968Z","shell.execute_reply.started":"2023-09-14T07:10:15.565707Z","shell.execute_reply":"2023-09-14T07:11:08.994740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df = pd.DataFrame(\n    {\"customer_ID\": _index, \"prediction\": _vals}\n).groupby(\"customer_ID\")[\"prediction\"].mean().reset_index()\n\nprint(res_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-14T01:57:48.937681Z","iopub.execute_input":"2023-09-14T01:57:48.938162Z","iopub.status.idle":"2023-09-14T01:57:48.944903Z","shell.execute_reply.started":"2023-09-14T01:57:48.938124Z","shell.execute_reply":"2023-09-14T01:57:48.943166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df.to_csv(\"submission.csv\", index=False)\nres_df.head()","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-09-14T01:49:20.361463Z","iopub.execute_input":"2023-09-14T01:49:20.361977Z","iopub.status.idle":"2023-09-14T01:49:20.382009Z","shell.execute_reply.started":"2023-09-14T01:49:20.361940Z","shell.execute_reply":"2023-09-14T01:49:20.381064Z"},"trusted":true},"execution_count":null,"outputs":[]}]}