{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# XGBoost Hyperparams Optimisation using Optuna\n\n\nIt is based on the [EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense).\nBased On - [LightGBM Quickstart](https://www.kaggle.com/code/ambrosm/amex-lightgbm-quickstart).","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\nfrom colorama import Fore, Back, Style\nimport gc\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.calibration import CalibrationDisplay\nfrom lightgbm import LGBMClassifier, log_evaluation\n\nplt.rcParams['axes.facecolor'] = '#0057b8' # blue\nplt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n                                         plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\nplt.rcParams['text.color'] = 'w'\n\nINFERENCE = True # set to False if you only want to cross-validate\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-30T10:54:57.09194Z","iopub.execute_input":"2022-05-30T10:54:57.092297Z","iopub.status.idle":"2022-05-30T10:55:00.119832Z","shell.execute_reply.started":"2022-05-30T10:54:57.092218Z","shell.execute_reply":"2022-05-30T10:55:00.118954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true, y_pred, return_components=False) -> float:\n    \"\"\"Amex metric for ndarrays\"\"\"\n    def top_four_percent_captured(df) -> float:\n        \"\"\"Corresponds to the recall for a threshold of 4 %\"\"\"\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(df) -> float:\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(df) -> float:\n        \"\"\"Corresponds to 2 * AUC - 1\"\"\"\n        df2 = pd.DataFrame({'target': df.target, 'prediction': df.target})\n        df2.sort_values('prediction', ascending=False, inplace=True)\n        return weighted_gini(df) / weighted_gini(df2)\n\n    df = pd.DataFrame({'target': y_true.ravel(), 'prediction': y_pred.ravel()})\n    df.sort_values('prediction', ascending=False, inplace=True)\n    g = normalized_weighted_gini(df)\n    d = top_four_percent_captured(df)\n\n    if return_components: return g, d, 0.5 * (g + d)\n    return 0.5 * (g + d)\n\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return 'amex', amex_metric(y_true, y_pred)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-30T10:55:00.121706Z","iopub.execute_input":"2022-05-30T10:55:00.122089Z","iopub.status.idle":"2022-05-30T10:55:00.13647Z","shell.execute_reply.started":"2022-05-30T10:55:00.122055Z","shell.execute_reply":"2022-05-30T10:55:00.135035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and preprocessing the data\n\nWe read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). Then we create two groups of features:\n- Selected features averaged over all statements of a customer\n- Selected features taken from the last statement of a customer\n\nThe code has been optimized for memory efficiency rather than readability. In particular, `.iloc[mask_array, columns]` needs much less RAM than the groupby construction used in the previous version of the notebook.\n\nPreprocessing for LightGBM is much simpler than for neural networks:\n1. Neural networks can't process missing values; LightGBM handles them automatically.\n1. Categorical features need to be one-hot encoded for neural networks; LightGBM handles them automatically.\n1. With neural networks, you need to think about outliers; tree-based algorithms deal with outliers easily.\n1. Neural networks need scaled inputs; tree-based algorithms ignore the scaling.","metadata":{}},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_1', 'B_11', 'B_16', 'B_17', 'B_18', 'B_2', 'B_20',\n                'B_28', 'B_3', 'B_4', 'B_5', 'B_7', 'B_9', 'D_112',\n                'D_121', 'D_141', 'D_39', 'D_41', 'D_42', 'D_43',\n                'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', \n                'D_50', 'D_51', 'D_53', 'D_54', 'D_56', 'D_58', \n                'D_59', 'D_60', 'D_91', 'P_2', 'P_3', 'R_1', 'R_2', \n                'R_27', 'R_3', 'R_7', 'S_11', 'S_26', 'S_3', 'S_5']\nfeatures_last = ['B_1', 'B_10', 'B_11', 'B_12', 'B_13', 'B_15', 'B_16',\n                 'B_17', 'B_18', 'B_19', 'B_2', 'B_20', 'B_22', 'B_23',\n                 'B_24', 'B_25', 'B_26', 'B_27', 'B_28', 'B_29', 'B_3',\n                 'B_32', 'B_33', 'B_36', 'B_38', 'B_39', 'B_4', 'B_40',\n                 'B_41', 'B_42', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9',\n                 'D_102', 'D_103', 'D_105', 'D_106', 'D_107', 'D_109',\n                 'D_112', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120',\n                 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_127', \n                 'D_129', 'D_132', 'D_133', 'D_135', 'D_136', 'D_137', \n                 'D_140', 'D_141', 'D_143', 'D_145', 'D_39', 'D_41',\n                 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48',\n                 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_54', 'D_55',\n                 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63',\n                 'D_64', 'D_66', 'D_70', 'D_72', 'D_73', 'D_74', 'D_75',\n                 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_82', 'D_83',\n                 'D_84', 'D_86', 'D_91', 'D_92', 'D_93', 'D_94', 'D_96',\n                 'P_2', 'P_3', 'R_1', 'R_10', 'R_11', 'R_12', 'R_13',\n                 'R_14', 'R_15', 'R_17', 'R_18', 'R_19', 'R_2', 'R_20', \n                 'R_21', 'R_22', 'R_24', 'R_25', 'R_26', 'R_27', 'R_3',\n                 'R_4', 'R_5', 'R_7', 'R_8', 'R_9', 'S_11', 'S_12',\n                 'S_13', 'S_15', 'S_17', 'S_20', 'S_22', 'S_23', \n                 'S_24', 'S_25', 'S_26', 'S_27', 'S_3', 'S_5', 'S_6',\n                 'S_7', 'S_8', 'S_9']\n\ntrain_test = [None, None] # first element is train, second element is test\nfor i in [1, 0] if INFERENCE else [0]:\n    train_test[i] = pd.read_feather(['../input/amexfeather/train_data.ftr',\n                                     '../input/amexfeather/test_data.ftr'][i])\n    cid = pd.Categorical(train_test[i].pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    if i == 0: # train\n        target = train_test[0].loc[last, 'target']\n    gc.collect()\n    print('Read', i)\n    df_avg = (train_test[i][features_avg]\n              .groupby(cid)\n              .mean()\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', i)\n    train_test[i] = (train_test[i].loc[last, features_last]\n                     .rename(columns={f: f\"{f}_last\" for f in features_last})\n                     .set_index(np.asarray(cid[last]))\n                    )\n    gc.collect()\n    print('Computed last', i)\n    train_test[i] = pd.concat([train_test[i], df_avg], axis=1)\n    del df_avg, cid, last\n\ntrain, test = tuple(train_test)\ndel train_test\nif INFERENCE: print('Shapes:', train.shape, target.shape, test.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:55:00.138245Z","iopub.execute_input":"2022-05-30T10:55:00.138722Z","iopub.status.idle":"2022-05-30T10:56:32.29227Z","shell.execute_reply.started":"2022-05-30T10:55:00.138684Z","shell.execute_reply":"2022-05-30T10:56:32.29095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross-validation\n\nWe cross-validate with a five-fold StratifiedKFold.\n\nNotice that lightgbm logs the validation score with the competition's scoring function every ten iterations.","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.294473Z","iopub.execute_input":"2022-05-30T10:56:32.295301Z","iopub.status.idle":"2022-05-30T10:56:32.390643Z","shell.execute_reply.started":"2022-05-30T10:56:32.295261Z","shell.execute_reply":"2022-05-30T10:56:32.389683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train['B_38_last']","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.391868Z","iopub.execute_input":"2022-05-30T10:56:32.392612Z","iopub.status.idle":"2022-05-30T10:56:32.397364Z","shell.execute_reply.started":"2022-05-30T10:56:32.392576Z","shell.execute_reply":"2022-05-30T10:56:32.396235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(['B_38_last', 'D_117_last', 'D_120_last', 'D_63_last', 'D_64_last', 'D_66_last'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.399074Z","iopub.execute_input":"2022-05-30T10:56:32.399627Z","iopub.status.idle":"2022-05-30T10:56:32.825545Z","shell.execute_reply.started":"2022-05-30T10:56:32.399589Z","shell.execute_reply":"2022-05-30T10:56:32.824511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.826787Z","iopub.execute_input":"2022-05-30T10:56:32.827939Z","iopub.status.idle":"2022-05-30T10:56:32.832106Z","shell.execute_reply.started":"2022-05-30T10:56:32.8279Z","shell.execute_reply":"2022-05-30T10:56:32.83126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial):\n    learning_rate = trial.suggest_float(\"learning_rate\", 1e-3, 0.2, log=True)\n    reg_lambda = trial.suggest_loguniform(\"reg_lambda\", 1e-2, 10.0)\n    reg_alpha = trial.suggest_loguniform(\"reg_alpha\", 18, 100.0)\n    subsample = trial.suggest_float(\"subsample\", 0.55, 1.0)\n    colsample_bytree = trial.suggest_float(\"colsample_bytree\", 0.01, 0.45)\n    gamma = trial.suggest_float(\"gamma\", 0.7,1.8)\n    max_depth = trial.suggest_int(\"max_depth\", 1, 7)\n    min_child_weight = trial.suggest_int(\"min_child_weight\", 3, 9)\n    n_estimators = trial.suggest_int(\"n_estimators\", 1000, 8000,step=200)\n    features = [f for f in train.columns if f != 'customer_ID' and f != 'target']\n    ONLY_FIRST_FOLD = False\n    \n    \n    \n    score_list = []\n    y_pred_list = []\n    kf = StratifiedKFold(n_splits=5)\n    for fold, (idx_tr, idx_va) in enumerate(kf.split(train, target)):\n        start_time = datetime.datetime.now()\n        \n        X_tr = train.iloc[idx_tr][features]\n        X_va = train.iloc[idx_va][features]\n        y_tr = target.iloc[idx_tr]\n        y_va = target.iloc[idx_va]\n        \n        \n        model = XGBClassifier(random_state=1,\n        tree_method=\"gpu_hist\",\n        gpu_id=0,\n        predictor=\"gpu_predictor\",\n        n_estimators=n_estimators,\n        learning_rate=learning_rate,\n        reg_lambda=reg_lambda,\n        reg_alpha=reg_alpha,\n        subsample=subsample,\n        colsample_bytree=colsample_bytree,\n        max_depth=max_depth,\n        gamma=gamma,\n                                  min_child_weight = min_child_weight,\n                                  booster = 'gbtree'\n    )\n\n        with warnings.catch_warnings():\n            warnings.filterwarnings('ignore', category=UserWarning)\n            model.fit(X_tr, y_tr,\n                      eval_set = [(X_tr, y_tr),(X_va, y_va)], verbose=20000\n                      )\n\n        y_va_pred = model.predict_proba(X_va)[:,1]\n        score = amex_metric(y_va.values, y_va_pred)\n        n_trees = model.get_booster().best_iteration\n        #if n_trees is None: n_trees = model.n_estimators\n        '''print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n              f\" {n_trees:5} trees |\"\n              f\"                Score = {score:.5f}{Style.RESET_ALL}\")'''\n        score_list.append(score)\n        \n\n        if ONLY_FIRST_FOLD: break # we only want the first fold\n\n    #print(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:                       {np.mean(score_list):.5f}{Style.RESET_ALL}\")\n    return np.mean(score_list)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.833502Z","iopub.execute_input":"2022-05-30T10:56:32.834336Z","iopub.status.idle":"2022-05-30T10:56:32.853283Z","shell.execute_reply.started":"2022-05-30T10:56:32.8343Z","shell.execute_reply":"2022-05-30T10:56:32.852358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# # Cross-validation of the classifier\n\n\n\n\n\n# def my_booster(random_state=1, n_estimators=195):\n#     return LGBMClassifier(n_estimators=n_estimators,\n#                           min_child_samples=2400,\n#                           num_leaves=127,\n#                           max_bins=511, random_state=random_state)\n\n# def xgb_boost():\n    \n    \n\n#     xgb_params = { 'random_state': 1, \n#     'n_jobs': 4,\n#     'booster': 'gbtree',\n#     'n_estimators': 4000,\n#     'learning_rate': 0.034682894846408095,\n#     'reg_lambda': 1.224383455634919,\n#     'reg_alpha': 36.043214512614476,\n#     'subsample': 0.9219010649982458,\n#     'colsample_bytree': 0.11247495917687526,\n#     'max_depth': 3,\n#     'min_child_weight': 6,\n#               'tree_method': 'gpu_hist',\n#                   'feval': lgb_amex_metric\n# }\n                         \n\n\n#     return XGBClassifier(**xgb_params)\n\n\n\n\n# #print(f\"{len(features)} features\")\n# score_list = []\n# y_pred_list = []\n# kf = StratifiedKFold(n_splits=5)\n# for fold, (idx_tr, idx_va) in enumerate(kf.split(train, target)):\n#     start_time = datetime.datetime.now()\n#     model = xgb_boost()\n#     X_tr = train.iloc[idx_tr][features]\n#     X_va = train.iloc[idx_va][features]\n#     y_tr = target.iloc[idx_tr]\n#     y_va = target.iloc[idx_va]\n    \n    \n#     with warnings.catch_warnings():\n#         warnings.filterwarnings('ignore', category=UserWarning)\n#         model.fit(X_tr, y_tr,\n#                   eval_set = [(X_tr, y_tr),(X_va, y_va)], verbose=2000\n#                   )\n                  \n#     y_va_pred = model.predict_proba(X_va)[:,1]\n#     score = amex_metric(y_va.values, y_va_pred)\n#     n_trees = model.get_booster().best_iteration\n#     if n_trees is None: n_trees = model.n_estimators\n#     print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n#           f\" {n_trees:5} trees |\"\n#           f\"                Score = {score:.5f}{Style.RESET_ALL}\")\n#     score_list.append(score)\n#     n_estimators += 75\n#     if INFERENCE:\n#         y_pred_list.append(model.predict_proba(test[features])[:,1])\n        \n#     if ONLY_FIRST_FOLD: break # we only want the first fold\n    \n# print(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:                       {np.mean(score_list):.5f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.854786Z","iopub.execute_input":"2022-05-30T10:56:32.855162Z","iopub.status.idle":"2022-05-30T10:56:32.865176Z","shell.execute_reply.started":"2022-05-30T10:56:32.855125Z","shell.execute_reply":"2022-05-30T10:56:32.864256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction histogram","metadata":{}},{"cell_type":"code","source":"%%time\nstudy = optuna.create_study(direction=\"maximize\",study_name = 'xgbclf1_')\nstudy.optimize(objective, n_trials=2)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T10:56:32.868415Z","iopub.execute_input":"2022-05-30T10:56:32.869008Z","iopub.status.idle":"2022-05-30T10:56:51.795355Z","shell.execute_reply.started":"2022-05-30T10:56:32.868882Z","shell.execute_reply":"2022-05-30T10:56:51.794547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calibration diagram\n\nThe calibration diagram shows how the model predicts the default probability of customers:","metadata":{}},{"cell_type":"code","source":"study.best_params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nWe submit the mean of the five predictions.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}