{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\nfrom colorama import Fore, Back, Style\nimport gc\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom lightgbm import LGBMClassifier\nimport lightgbm as lgb\n\nplt.rcParams['axes.facecolor'] = '#0057b8' # blue\nplt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n                                         plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\nplt.rcParams['text.color'] = 'w'\n\nINFERENCE = True # set to False if you only want to cross-validate\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2022-07-09T08:35:41.166942Z","iopub.status.busy":"2022-07-09T08:35:41.166744Z","iopub.status.idle":"2022-07-09T08:35:42.607561Z","shell.execute_reply":"2022-07-09T08:35:42.606737Z","shell.execute_reply.started":"2022-07-09T08:35:41.166892Z"},"papermill":{"duration":2.120406,"end_time":"2022-06-19T17:18:11.703947","exception":false,"start_time":"2022-06-19T17:18:09.583541","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"# @yunchonggan's fast metric implementation\n# From https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\ndef amex_metric(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by descring prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)\n\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(y_true, y_pred),\n            True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2022-07-09T08:35:42.608470Z","iopub.status.busy":"2022-07-09T08:35:42.608342Z","iopub.status.idle":"2022-07-09T08:35:42.613569Z","shell.execute_reply":"2022-07-09T08:35:42.613199Z","shell.execute_reply.started":"2022-07-09T08:35:42.608455Z"},"papermill":{"duration":0.019778,"end_time":"2022-06-19T17:18:11.729193","exception":false,"start_time":"2022-06-19T17:18:11.709415","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeatures_avg = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_53', 'D_54', 'D_55', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_65', 'D_66', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_86', 'D_91', 'D_92', 'D_94', 'D_96', 'D_103', 'D_104', 'D_108', 'D_112', 'D_113', 'D_114', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_129', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_14', 'R_15', 'R_16', 'R_17', 'R_20', 'R_21', 'R_22', 'R_24', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_9', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_18', 'S_22', 'S_23', 'S_25', 'S_26']\nfeatures_min = ['B_2', 'B_4', 'B_5', 'B_9', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_19', 'B_20', 'B_28', 'B_29', 'B_33', 'B_36', 'B_42', 'D_39', 'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', 'D_53', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_71', 'D_74', 'D_75', 'D_78', 'D_83', 'D_102', 'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', 'D_132', 'D_140', 'D_141', 'D_144', 'D_145', 'P_2', 'P_3', 'R_1', 'R_27', 'S_3', 'S_5', 'S_7', 'S_9', 'S_11', 'S_12', 'S_23', 'S_25']\nfeatures_max = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_21', 'B_23', 'B_24', 'B_25', 'B_29', 'B_30', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_52', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_63', 'D_64', 'D_65', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_91', 'D_102', 'D_105', 'D_107', 'D_110', 'D_111', 'D_112', 'D_115', 'D_116', 'D_117', 'D_118', 'D_119', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_138', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_3', 'R_5', 'R_6', 'R_7', 'R_8', 'R_10', 'R_11', 'R_14', 'R_17', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_7', 'S_8', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\nfeatures_last = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_26', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_36', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_54', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63', 'D_64', 'D_65', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_75', 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_81', 'D_82', 'D_83', 'D_86', 'D_91', 'D_96', 'D_105', 'D_106', 'D_112', 'D_114', 'D_119', 'D_120', 'D_121', 'D_122', 'D_124', 'D_125', 'D_126', 'D_127', 'D_130', 'D_131', 'D_132', 'D_133', 'D_134', 'D_138', 'D_140', 'D_141', 'D_142', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_4', 'R_5', 'R_6', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_12', 'R_13', 'R_14', 'R_15', 'R_19', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_8', 'S_9', 'S_11', 'S_12', 'S_13', 'S_16', 'S_19', 'S_20', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\n\nfor i in ['test', 'train'] if INFERENCE else ['train']:\n    df = pd.read_parquet(f'../input/amex-data-integer-dtypes-parquet-format/{i}.parquet')\n    cid = pd.Categorical(df.pop('customer_ID'), ordered=True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    if 'target' in df.columns:\n        df.drop(columns=['target'], inplace=True)\n    gc.collect()\n    print('Read', i)\n    df_avg = (df\n              .groupby(cid)\n              .mean()[features_avg]\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', i)\n    df_min = (df\n              .groupby(cid)\n              .min()[features_min]\n              .rename(columns={f: f\"{f}_min\" for f in features_min})\n             )\n    gc.collect()\n    print('Computed min', i)\n    df_max = (df\n              .groupby(cid)\n              .max()[features_max]\n              .rename(columns={f: f\"{f}_max\" for f in features_max})\n             )\n    gc.collect()\n    print('Computed max', i)\n    df = (df.loc[last, features_last]\n          .rename(columns={f: f\"{f}_last\" for f in features_last})\n          .set_index(np.asarray(cid[last]))\n         )\n    gc.collect()\n    print('Computed last', i)\n    df = pd.concat([df, df_min, df_max, df_avg], axis=1)\n    if i == 'train': train = df\n    else: test = df\n    print(f\"{i} shape: {df.shape}\")\n    del df, df_avg, df_min, df_max, cid, last\n\ntarget = pd.read_csv('../input/amex-default-prediction/train_labels.csv').target.values\nprint(f\"target shape: {target.shape}\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:35:42.614843Z","iopub.status.busy":"2022-07-09T08:35:42.614700Z","iopub.status.idle":"2022-07-09T08:39:22.754724Z","shell.execute_reply":"2022-07-09T08:39:22.754285Z","shell.execute_reply.started":"2022-07-09T08:35:42.614828Z"},"papermill":{"duration":512.612147,"end_time":"2022-06-19T17:26:44.356584","exception":false,"start_time":"2022-06-19T17:18:11.744437","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [\n    \"B_30\",\n    \"B_38\",\n    \"D_114\",\n    \"D_116\",\n    \"D_117\",\n    \"D_120\",\n    \"D_126\",\n    \"D_63\",\n    \"D_64\",\n    \"D_66\",\n    \"D_68\",\n]\n\nlast_cat_features = []\nfor col in train.columns:\n    if '_'.join(col.split('_')[:2]) in cat_features:\n        if 'max' in col or 'last' in col:\n            last_cat_features.append(col)\n        else:\n            print(col)","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:39:22.755583Z","iopub.status.busy":"2022-07-09T08:39:22.755399Z","iopub.status.idle":"2022-07-09T08:39:22.759919Z","shell.execute_reply":"2022-07-09T08:39:22.759586Z","shell.execute_reply.started":"2022-07-09T08:39:22.755567Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in last_cat_features:\n    if train[col].min() < 0:\n        train[col] += 1\n        test[col] += 1","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:39:22.760582Z","iopub.status.busy":"2022-07-09T08:39:22.760429Z","iopub.status.idle":"2022-07-09T08:39:22.860851Z","shell.execute_reply":"2022-07-09T08:39:22.860161Z","shell.execute_reply.started":"2022-07-09T08:39:22.760567Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LightGBM","metadata":{"papermill":{"duration":0.005407,"end_time":"2022-06-19T17:26:44.367938","exception":false,"start_time":"2022-06-19T17:26:44.362531","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n# Cross-validation of the classifier\n\nONLY_FIRST_FOLD = False\n\nfeatures = [f for f in train.columns if f != 'customer_ID' and f != 'target']\n\ndef my_booster(random_state=1, n_estimators=1200):\n    return LGBMClassifier(n_estimators=n_estimators,\n                          learning_rate=0.03, reg_lambda=50,\n                          min_child_samples=2400,\n                          colsample_bytree=0.19,\n                          max_bins=511, random_state=random_state)\n      \nprint(f\"{len(features)} features\")\nscore_list = []\ny_lgb_pred_list = []\nkf = StratifiedKFold(n_splits=5)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train, target)):\n    X_tr, X_va, y_tr, y_va, model = None, None, None, None, None\n    start_time = datetime.datetime.now()\n    X_tr = train.iloc[idx_tr][features]\n    X_va = train.iloc[idx_va][features]\n    y_tr = target[idx_tr]\n    y_va = target[idx_va]\n    \n    model = my_booster()\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore', category=UserWarning)\n        model.fit(X_tr, y_tr,\n                  eval_set = [(X_va, y_va)], \n                  eval_metric=[lgb_amex_metric],\n                  callbacks=[lgb.log_evaluation(100)],\n                 categorical_feature=last_cat_features)\n    X_tr, y_tr = None, None\n    y_va_pred = model.predict_proba(X_va, raw_score=True)\n    score = amex_metric(y_va, y_va_pred)\n    n_trees = model.best_iteration_\n    if n_trees is None: n_trees = model.n_estimators\n    print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {score:.5f}{Style.RESET_ALL}\")\n    score_list.append(score)\n    \n    if INFERENCE:\n        y_lgb_pred_list.append(model.predict_proba(test[features], raw_score=True))\n        \n    if ONLY_FIRST_FOLD: break # we only want the first fold\n    \nprint(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:                       {np.mean(score_list):.5f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:39:22.861726Z","iopub.status.busy":"2022-07-09T08:39:22.861590Z","iopub.status.idle":"2022-07-09T08:46:50.113964Z","shell.execute_reply":"2022-07-09T08:46:50.113571Z","shell.execute_reply.started":"2022-07-09T08:39:22.861710Z"},"papermill":{"duration":1943.230965,"end_time":"2022-06-19T17:59:07.604475","exception":false,"start_time":"2022-06-19T17:26:44.373510","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"for col in last_cat_features:\n    train[col] = train[col].astype(\"category\")\n    test[col] = test[col].astype(\"category\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T09:11:23.849259Z","iopub.status.busy":"2022-07-09T09:11:23.848708Z","iopub.status.idle":"2022-07-09T09:11:24.012837Z","shell.execute_reply":"2022-07-09T09:11:24.012106Z","shell.execute_reply.started":"2022-07-09T09:11:23.849209Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in last_cat_features:\n    train[col] = train[col].astype(\"int\")\n    test[col] = test[col].astype(\"int\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T09:11:24.018588Z","iopub.status.busy":"2022-07-09T09:11:24.018395Z","iopub.status.idle":"2022-07-09T09:11:24.366662Z","shell.execute_reply":"2022-07-09T09:11:24.365948Z","shell.execute_reply.started":"2022-07-09T09:11:24.018570Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Cross-validation of the classifier\n\nONLY_FIRST_FOLD = False\n\nfeatures = [f for f in train.columns if f != 'customer_ID' and f != 'target']\n\ndef xgb_amex_metric(y_pred, dtrain):\n    labels = dtrain.get_label()\n    \"\"\"The competition metric with xgboost's calling convention\"\"\"\n    return ('amex',\n            amex_metric(labels, y_pred))\n\nimport xgboost as xgb\ndef my_booster(random_state=1, n_estimators=1500):\n    return xgb.XGBClassifier(\n        tree_method=\"gpu_hist\",\n        # enable_categorical=True,\n        \n        min_child_weight=1,\n        verbosity=0,\n        n_estimators=n_estimators,\n        n_jobs=-1,\n        learning_rate=0.03,\n        random_state=random_state,\n    )\n      \nprint(f\"{len(features)} features\")\nscore_list = []\ny_xgb_pred_list = []\nkf = StratifiedKFold(n_splits=5)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train, target)):\n    X_tr, X_va, y_tr, y_va, model = None, None, None, None, None\n    start_time = datetime.datetime.now()\n    X_tr = train.iloc[idx_tr][features]\n    X_va = train.iloc[idx_va][features]\n    y_tr = target[idx_tr]\n    y_va = target[idx_va]\n    \n    model = my_booster()\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore', category=UserWarning)\n        model.fit(X_tr, y_tr,\n                  eval_set = [(X_va, y_va)], \n                  # eval_metric=[lgb_amex_metric],\n                  callbacks=[xgb.callback.EarlyStopping(\n                    rounds=100,\n                    save_best=True,\n                    maximize=True,\n                    data_name=\"validation_0\",\n                    metric_name=\"amex\",\n                )], verbose=False,\n                  eval_metric=xgb_amex_metric\n        )\n    X_tr, y_tr = None, None\n    y_va_pred = model.predict_proba(X_va)[:, 1]\n    score = amex_metric(y_va, y_va_pred)\n    n_trees = model.best_iteration\n    if n_trees is None: n_trees = model.n_estimators\n    print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {score:.5f}{Style.RESET_ALL}\")\n    score_list.append(score)\n    \n    if INFERENCE:\n        y_xgb_pred_list.append(model.predict_proba(test[features])[:, 1])\n        \n    if ONLY_FIRST_FOLD: break # we only want the first fold\n    \nprint(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:                       {np.mean(score_list):.5f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T09:11:25.849598Z","iopub.status.busy":"2022-07-09T09:11:25.849203Z","iopub.status.idle":"2022-07-09T09:17:34.244868Z","shell.execute_reply":"2022-07-09T09:17:34.244441Z","shell.execute_reply.started":"2022-07-09T09:11:25.849575Z"},"scrolled":true,"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CatBoost","metadata":{}},{"cell_type":"code","source":"for col in last_cat_features:\n    train[col] = train[col].astype(\"int\")\n    test[col] = test[col].astype(\"int\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:53:00.471797Z","iopub.status.busy":"2022-07-09T08:53:00.471597Z","iopub.status.idle":"2022-07-09T08:53:00.549172Z","shell.execute_reply":"2022-07-09T08:53:00.548665Z","shell.execute_reply.started":"2022-07-09T08:53:00.471778Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Cross-validation of the classifier\n\nONLY_FIRST_FOLD = False\n\nfeatures = [f for f in train.columns if f != 'customer_ID' and f != 'target']\n\n\nfrom catboost import CatBoostClassifier, Pool\n\ndef my_booster(random_state=1, n_estimators=2200):\n    return CatBoostClassifier(\n        iterations=n_estimators,\n        task_type=\"GPU\",\n        min_data_in_leaf=64,\n        learning_rate=0.01, loss_function=\"Logloss\", verbose=False\n    )\n      \nprint(f\"{len(features)} features\")\nscore_list = []\ny_cab_pred_list = []\nkf = StratifiedKFold(n_splits=5)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train, target)):\n    X_tr, X_va, y_tr, y_va, model = None, None, None, None, None\n    start_time = datetime.datetime.now()\n    X_tr = train.iloc[idx_tr][features]\n    X_va = train.iloc[idx_va][features]\n    y_tr = target[idx_tr]\n    y_va = target[idx_va]\n    \n    model = my_booster()\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore', category=UserWarning)\n        model.fit(X_tr, y_tr, cat_features=last_cat_features,\n                  eval_set = Pool(X_va, y_va, cat_features=last_cat_features), \n        )\n    X_tr, y_tr = None, None\n    y_va_pred = model.predict_proba(X_va)[:, 1]\n    score = amex_metric(y_va, y_va_pred)\n    n_trees = model.get_best_iteration()\n    if n_trees is None: n_trees = model.n_estimators\n    print(f\"{Fore.GREEN}{Style.BRIGHT}Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {score:.5f}{Style.RESET_ALL}\")\n    score_list.append(score)\n    \n    if INFERENCE:\n        y_cab_pred_list.append(model.predict_proba(test[features])[:, 1])\n        \n    if ONLY_FIRST_FOLD: break # we only want the first fold\n    \nprint(f\"{Fore.GREEN}{Style.BRIGHT}OOF Score:                       {np.mean(score_list):.5f}{Style.RESET_ALL}\")","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:53:00.550149Z","iopub.status.busy":"2022-07-09T08:53:00.549902Z","iopub.status.idle":"2022-07-09T08:58:59.581120Z","shell.execute_reply":"2022-07-09T08:58:59.580618Z","shell.execute_reply.started":"2022-07-09T08:53:00.550131Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.010863,"end_time":"2022-06-19T17:59:08.786989","exception":false,"start_time":"2022-06-19T17:59:08.776126","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sub = pd.DataFrame({'customer_ID': test.index,\n                    'prediction': np.mean(y_lgb_pred_list, axis=0)})\nsub.to_csv('submission_lgb.csv', index=False)\n\nsub = pd.DataFrame({'customer_ID': test.index,\n                    'prediction': np.mean(y_xgb_pred_list, axis=0)})\nsub.to_csv('submission_xgb.csv', index=False)\n\nsub = pd.DataFrame({'customer_ID': test.index,\n                    'prediction': np.mean(y_cab_pred_list, axis=0)})\nsub.to_csv('submission_cab.csv', index=False)","metadata":{"execution":{"iopub.execute_input":"2022-07-09T08:58:59.582226Z","iopub.status.busy":"2022-07-09T08:58:59.581904Z","iopub.status.idle":"2022-07-09T08:59:08.119401Z","shell.execute_reply":"2022-07-09T08:59:08.118323Z","shell.execute_reply.started":"2022-07-09T08:58:59.582208Z"},"papermill":{"duration":4.84095,"end_time":"2022-06-19T17:59:13.638945","exception":false,"start_time":"2022-06-19T17:59:08.797995","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}