{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LightGBM Quickstart for the *American Express - Default Prediction* competition\n\nThis notebook shows how to apply LightGBM to the competition data, and it introduces a space-efficient way of feature engineering.\n\nIt is based on the [EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense).","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\nfrom colorama import Fore, Back, Style\nimport gc\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.calibration import CalibrationDisplay\nfrom lightgbm import LGBMClassifier, log_evaluation\nfrom imblearn.over_sampling import SMOTE\n\nplt.rcParams['axes.facecolor'] = '#0057b8' # blue\nplt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n                                         plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\nplt.rcParams['text.color'] = 'w'\n\nINFERENCE = True # set to False if you only want to cross-validate\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T16:02:49.750050Z","iopub.execute_input":"2022-05-28T16:02:49.750676Z","iopub.status.idle":"2022-05-28T16:02:52.904783Z","shell.execute_reply.started":"2022-05-28T16:02:49.750549Z","shell.execute_reply":"2022-05-28T16:02:52.903765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true, y_pred, return_components=False) -> float:\n    \"\"\"Amex metric for ndarrays\"\"\"\n    def top_four_percent_captured(df) -> float:\n        \"\"\"Corresponds to the recall for a threshold of 4 %\"\"\"\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(df) -> float:\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(df) -> float:\n        \"\"\"Corresponds to 2 * AUC - 1\"\"\"\n        df2 = pd.DataFrame({'target': df.target, 'prediction': df.target})\n        df2.sort_values('prediction', ascending=False, inplace=True)\n        return weighted_gini(df) / weighted_gini(df2)\n\n    df = pd.DataFrame({'target': y_true.ravel(), 'prediction': y_pred.ravel()})\n    df.sort_values('prediction', ascending=False, inplace=True)\n    g = normalized_weighted_gini(df)\n    d = top_four_percent_captured(df)\n\n    if return_components: return g, d, 0.5 * (g + d)\n    return 0.5 * (g + d)\n\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(y_true, y_pred),\n            True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T16:02:52.906742Z","iopub.execute_input":"2022-05-28T16:02:52.907156Z","iopub.status.idle":"2022-05-28T16:02:52.924780Z","shell.execute_reply.started":"2022-05-28T16:02:52.907106Z","shell.execute_reply":"2022-05-28T16:02:52.923217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and preprocessing the data\n\nWe read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). Then we create two groups of features:\n- Selected features averaged over all statements of a customer\n- Selected features taken from the last statement of a customer\n\nThe code has been optimized for memory efficiency rather than readability. In particular, `.iloc[mask_array, columns]` needs much less RAM than the groupby construction used in the previous version of the notebook.\n\nPreprocessing for LightGBM is much simpler than for neural networks:\n1. Neural networks can't process missing values; LightGBM handles them automatically.\n1. Categorical features need to be one-hot encoded for neural networks; LightGBM handles them automatically.\n1. With neural networks, you need to think about outliers; tree-based algorithms deal with outliers easily.\n1. Neural networks need scaled inputs; tree-based algorithms ignore the scaling.","metadata":{}},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_1', 'B_11', 'B_16', 'B_17', 'B_18', 'B_2', 'B_20',\n                'B_28', 'B_3', 'B_4', 'B_5', 'B_7', 'B_9', 'D_112',\n                'D_121', 'D_141', 'D_39', 'D_41', 'D_42', 'D_43',\n                'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', \n                'D_50', 'D_51', 'D_53', 'D_54', 'D_56', 'D_58', \n                'D_59', 'D_60', 'D_91', 'P_2', 'P_3', 'R_1', 'R_2', \n                'R_27', 'R_3', 'R_7', 'S_11', 'S_26', 'S_3', 'S_5']\nfeatures_last = ['B_1', 'B_10', 'B_11', 'B_12', 'B_13', 'B_15', 'B_16',\n                 'B_17', 'B_18', 'B_19', 'B_2', 'B_20', 'B_22', 'B_23',\n                 'B_24', 'B_25', 'B_26', 'B_27', 'B_28', 'B_29', 'B_3',\n                 'B_32', 'B_33', 'B_36', 'B_38', 'B_39', 'B_4', 'B_40',\n                 'B_41', 'B_42', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9',\n                 'D_102', 'D_103', 'D_105', 'D_106', 'D_107', 'D_109',\n                 'D_112', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120',\n                 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_127', \n                 'D_129', 'D_132', 'D_133', 'D_135', 'D_136', 'D_137', \n                 'D_140', 'D_141', 'D_143', 'D_145', 'D_39', 'D_41',\n                 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48',\n                 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_54', 'D_55',\n                 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63',\n                 'D_64', 'D_66', 'D_70', 'D_72', 'D_73', 'D_74', 'D_75',\n                 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_82', 'D_83',\n                 'D_84', 'D_86', 'D_91', 'D_92', 'D_93', 'D_94', 'D_96',\n                 'P_2', 'P_3', 'R_1', 'R_10', 'R_11', 'R_12', 'R_13',\n                 'R_14', 'R_15', 'R_17', 'R_18', 'R_19', 'R_2', 'R_20', \n                 'R_21', 'R_22', 'R_24', 'R_25', 'R_26', 'R_27', 'R_3',\n                 'R_4', 'R_5', 'R_7', 'R_8', 'R_9', 'S_11', 'S_12',\n                 'S_13', 'S_15', 'S_17', 'S_20', 'S_22', 'S_23', \n                 'S_24', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_6',\n                 'S_7', 'S_8', 'S_9']\n\ntrain_test = [None, None] # first element is train, second element is test\nfor i in [1, 0] if INFERENCE else [0]:\n    train_test[i] = pd.read_feather(['../input/amexfeather/train_data.ftr',\n                                     '../input/amexfeather/test_data.ftr'][i])\n    cid = pd.Categorical(train_test[i].pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    if i == 0: # train\n        target = train_test[0].loc[last, 'target']\n    gc.collect()\n    print('Read', i)\n    df_avg = (train_test[i][features_avg]\n              .groupby(cid)\n              .mean()\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', i)\n    train_test[i] = (train_test[i].loc[last, features_last]\n                     .rename(columns={f: f\"{f}_last\" for f in features_last})\n                     .set_index(np.asarray(cid[last]))\n                    )\n    gc.collect()\n    print('Computed last', i)\n    train_test[i] = pd.concat([train_test[i], df_avg], axis=1)\n    del df_avg, cid, last\n\ntrain, test = tuple(train_test)\ndel train_test\nif INFERENCE: print('Shapes:', train.shape, target.shape, test.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:02:52.926166Z","iopub.execute_input":"2022-05-28T16:02:52.926864Z","iopub.status.idle":"2022-05-28T16:04:22.548440Z","shell.execute_reply.started":"2022-05-28T16:02:52.926817Z","shell.execute_reply":"2022-05-28T16:04:22.547119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross-validation\n\nWe cross-validate with a five-fold StratifiedKFold.\n\nNotice that lightgbm logs the validation score with the competition's scoring function every ten iterations.","metadata":{}},{"cell_type":"code","source":"train.select_dtypes(include=['category']).dtypes","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:04:22.550359Z","iopub.execute_input":"2022-05-28T16:04:22.551377Z","iopub.status.idle":"2022-05-28T16:04:22.692965Z","shell.execute_reply.started":"2022-05-28T16:04:22.551338Z","shell.execute_reply":"2022-05-28T16:04:22.691964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get Index for Multiple Column Labels/Names\nquery_cols=['B_38_last',\n          'D_117_last',\n          'D_120_last',\n          'D_63_last',\n          'D_64_last',\n          'D_66_last'\n         ]\ncols_index = [train.columns.get_loc(col) for col in query_cols]\nprint(cols_index)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:04:22.696499Z","iopub.execute_input":"2022-05-28T16:04:22.696880Z","iopub.status.idle":"2022-05-28T16:04:22.702831Z","shell.execute_reply.started":"2022-05-28T16:04:22.696849Z","shell.execute_reply":"2022-05-28T16:04:22.701681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['D_66_last'].fillna(0, inplace=True)\ntrain['D_120_last'].fillna(0, inplace=True)\ntrain['D_117_last'] = pd.Categorical(train['D_117_last'], \n                                     categories = [-1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0])\ntrain['D_117_last'].fillna(0.0, inplace=True)\ntrain['B_38_last'] = pd.Categorical(train['B_38_last'], \n                                     categories = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])\ntrain['B_38_last'].fillna(0.0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:04:22.704355Z","iopub.execute_input":"2022-05-28T16:04:22.704861Z","iopub.status.idle":"2022-05-28T16:04:22.731407Z","shell.execute_reply.started":"2022-05-28T16:04:22.704813Z","shell.execute_reply":"2022-05-28T16:04:22.730227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[list(train.select_dtypes(include=['float16']).dtypes.index)] = train[list(train.select_dtypes(include=['float16']).dtypes.index)].fillna(0, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:04:22.733387Z","iopub.execute_input":"2022-05-28T16:04:22.734229Z","iopub.status.idle":"2022-05-28T16:04:24.360952Z","shell.execute_reply.started":"2022-05-28T16:04:22.734173Z","shell.execute_reply":"2022-05-28T16:04:24.359795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTENC\n\nsmote_over_sample = SMOTENC(categorical_features=cols_index,\n                            sampling_strategy='minority')\nlabels = target.tolist()\ntrain, target = smote_over_sample.fit_resample(train, target)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:04:24.362447Z","iopub.execute_input":"2022-05-28T16:04:24.362931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Cross-validation of the classifier\n\nONLY_FIRST_FOLD = False\n\nfeatures = [f for f in train.columns if f != 'customer_ID' and f != 'target']\n\ndef my_booster(random_state=1, n_estimators=195):\n    return LGBMClassifier(n_estimators=n_estimators,\n                          min_child_samples=2400,\n                          num_leaves=127,\n                          max_bins=511, random_state=random_state)\n      \nprint(f\"{len(features)} features\")\nscore_list = []\ny_pred_list = []\nkf = StratifiedKFold(n_splits=5)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train, target)):\n    start_time = datetime.datetime.now()\n    X_tr = train.iloc[idx_tr][features]\n    X_va = train.iloc[idx_va][features]\n    y_tr = target.iloc[idx_tr]\n    y_va = target.iloc[idx_va]\n    \n    model = my_booster()\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore', category=UserWarning)\n        model.fit(X_tr, y_tr,\n                  eval_set = [(X_tr, y_tr),(X_va, y_va)], \n                  eval_metric=[lgb_amex_metric],\n                  callbacks=[log_evaluation(10)])\n    y_va_pred = model.predict_proba(X_va)[:,1]\n    score = amex_metric(y_va.values, y_va_pred)\n    n_trees = model.best_iteration_\n    if n_trees is None: n_trees = model.n_estimators\n    print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {score:.5f}{Style.RESET_ALL}\")\n    score_list.append(score)\n    \n    if INFERENCE:\n        y_pred_list.append(model.predict_proba(test[features])[:,1])\n        \n    if ONLY_FIRST_FOLD: break # we only want the first fold\n    \nprint(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:                       {np.mean(score_list):.5f}{Style.RESET_ALL}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction histogram","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 4))\nplt.hist(y_va_pred[y_va == 0], bins=np.linspace(0, 1, 21),\n         alpha=0.5, density=True, label='0')\nplt.hist(y_va_pred[y_va == 1], bins=np.linspace(0, 1, 21),\n         alpha=0.5, density=True, label='1')\nplt.xlabel('y_pred')\nplt.ylabel('density')\nplt.title('OOF Prediction histogram', color='k')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calibration diagram\n\nThe calibration diagram shows how the model predicts the default probability of customers:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nCalibrationDisplay.from_predictions(y_va, y_va_pred, n_bins=50, strategy='quantile', ax=plt.gca())\nplt.title('Probability calibration')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nWe submit the mean of the five predictions.","metadata":{}},{"cell_type":"code","source":"if INFERENCE:\n    sub = pd.DataFrame({'customer_ID': test.index,\n                        'prediction': np.mean(y_pred_list, axis=0)})\n    sub.to_csv('submission.csv', index=False)\n    display(sub)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}