{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AMEX - lgbm + Feature Engineering\n\nBaseline lgbm approach + blanket feature eng.\n\nThe magic (Feature Engineering) is here: https://www.kaggle.com/code/lucasmorin/amex-feature-engineering\n\nSome stuff from my G-research lgbm baseline: https://www.kaggle.com/code/lucasmorin/online-fe-lgbm-feval-importances\n\nAnd my JPX XGBRanker baseline: https://www.kaggle.com/code/lucasmorin/jpx-all-data-xgbranker\n\nSome other stuff from @ambrosm lgbm baseline here: https://www.kaggle.com/code/ambrosm/amex-lightgbm-quickstart\n\n**This Notebook is part of a serie built for the AMEX competition:**\n- [Base Feature engineering](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering-base)\n- [Baseline lgbm](https://www.kaggle.com/code/lucasmorin/amex-lgbm-features-eng)\n- [Feature Engineering 2: aggregation function](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering-2-aggreg-functions)\n- [Feature Engineering 3: transformation function](https://www.kaggle.com/code/lucasmorin/amex-feature-engineering-3-transform-functions)\n\n**With associated Data Sets:**\n- [Base Feature engineering](https://www.kaggle.com/datasets/lucasmorin/amex-base-fe)\n- [Feature Engineering 2 - aggregation function](https://www.kaggle.com/datasets/lucasmorin/amex-fe2)\n- [Feature Engineering 3 - transform function](https://www.kaggle.com/datasets/lucasmorin/amex-fe3)\n\n\n\n**Please make sure to upvote everything you use / find interesting / usefull**","metadata":{"papermill":{"duration":0.00979,"end_time":"2022-01-02T19:15:51.734235","exception":false,"start_time":"2022-01-02T19:15:51.724445","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport pickle\n\nimport time\nfrom datetime import datetime\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport lightgbm as lgb\n\nimport warnings\n\nfrom sklearn.model_selection import StratifiedKFold\n\nbase_seed = 0\n\nDEBUG = False\nTEST = True\n\nif ~DEBUG:\n    warnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":2.402335,"end_time":"2022-01-02T19:15:54.14708","exception":false,"start_time":"2022-01-02T19:15:51.744745","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T06:44:08.283593Z","iopub.execute_input":"2022-07-31T06:44:08.285689Z","iopub.status.idle":"2022-07-31T06:44:10.996444Z","shell.execute_reply.started":"2022-07-31T06:44:08.285552Z","shell.execute_reply":"2022-07-31T06:44:10.995466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_pickle('../input/amex-feature-engineering/train_data_agg.pkl')\ntrain_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv').set_index('customer_ID').loc[train_data.index]\n\nFeatures = train_data.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:44:10.998033Z","iopub.execute_input":"2022-07-31T06:44:10.998392Z","iopub.status.idle":"2022-07-31T06:44:26.914800Z","shell.execute_reply.started":"2022-07-31T06:44:10.998359Z","shell.execute_reply":"2022-07-31T06:44:26.913799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n\n# from ambrosm notebook\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(pd.DataFrame({'target': y_true}), pd.Series(y_pred, name='prediction')),\n            True)\n\n#see : https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534\ndef amex_metric_mod_lgbm(y_pred: np.ndarray, data: lgb.Dataset):\n\n    y_true = data.get_label()\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 'AMEX', 0.5 * (gini[1]/gini[0]+ top_four), True","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:44:26.916202Z","iopub.execute_input":"2022-07-31T06:44:26.916823Z","iopub.status.idle":"2022-07-31T06:44:26.938642Z","shell.execute_reply.started":"2022-07-31T06:44:26.916786Z","shell.execute_reply":"2022-07-31T06:44:26.937284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_fold = 2 if DEBUG else 3\nn_seed = 1 if TEST else (2 if DEBUG else 3)\nn_estimators = 100 if DEBUG else 10000\n\nkf = StratifiedKFold(n_splits=n_fold)\n\nimportances = []\nimportances_split = []\nmodels = {}\ndf_scores = []\n\nids_folds = {}\npreds_tr_va = {}\n\nSAMPLE = False\n\n\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train_data, train_labels)):\n    \n    if TEST:\n        if fold>0:\n            continue\n    \n    ids_folds[fold] = (idx_tr, idx_va)\n    \n    X_tr = train_data[Features].iloc[idx_tr]\n    X_va = train_data[Features].iloc[idx_va]\n    y_tr = train_labels.iloc[idx_tr]\n    y_va = train_labels.iloc[idx_va]\n    \n    lgb_train_data = lgb.Dataset(X_tr, label=y_tr)\n    lgb_val_data = lgb.Dataset(X_va, label=y_va)\n    \n    for seed in range(n_seed):\n        print('Fold: '+str(fold)+ ' - seed: '+str(seed))\n        key = str(fold)+'-'+str(seed)\n        \n        parameters = {\n            'objective': 'cross_entropy_lambda',\n            'boosting': 'dart',\n            'learning_rate': 0.005,\n            #'min_child_samples': 1000,\n            'reg_alpha':10,\n            'feature_fraction':0.3,\n            'bagging_fraction':0.3,\n            'max_depth': 6,\n            'seed':seed,\n            'n_estimators':n_estimators,\n            'verbose':-1,\n            'linear_tree': True\n        }\n\n        clf = lgb.train(parameters,\n                               lgb_train_data,\n                               valid_sets = [lgb_train_data,lgb_val_data],\n                               verbose_eval = 100,\n                               feval=amex_metric_mod_lgbm,\n                               early_stopping_rounds=200)\n\n        \n        preds_tr = pd.Series(clf.predict(X_tr)).rename('prediction')\n        preds_va = pd.Series(clf.predict(X_va)).rename('prediction')\n        \n        preds_tr_va[(fold, seed)] = (preds_tr, preds_va)\n        \n        score = amex_metric(y_va.reset_index(drop=True), preds_va)\n        models[key] = clf\n        df_scores.append((fold, seed, score))\n        print(f'Fold: {fold} - seed: {seed} - score {score:.2%}')\n        importances.append(clf.feature_importance(importance_type='gain'))\n        importances_split.append(clf.feature_importance(importance_type='split'))","metadata":{"papermill":{"duration":1432.413344,"end_time":"2022-01-02T19:39:46.616263","exception":false,"start_time":"2022-01-02T19:15:54.202919","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T06:44:26.940688Z","iopub.execute_input":"2022-07-31T06:44:26.941180Z","iopub.status.idle":"2022-07-31T06:46:30.574603Z","shell.execute_reply.started":"2022-07-31T06:44:26.941143Z","shell.execute_reply":"2022-07-31T06:46:30.573464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle.dump(ids_folds, open('ids_folds.p', 'wb'))\npickle.dump(preds_tr_va, open('preds_tr_va.p', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:30.576060Z","iopub.execute_input":"2022-07-31T06:46:30.576627Z","iopub.status.idle":"2022-07-31T06:46:30.590489Z","shell.execute_reply.started":"2022-07-31T06:46:30.576582Z","shell.execute_reply":"2022-07-31T06:46:30.589701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_results = pd.DataFrame(df_scores,columns=['fold','seed','score']).pivot(index='fold',columns='seed',values='score')\n\ndf_results.loc['seed_mean']= df_results.mean(numeric_only=True, axis=0)\ndf_results.loc[:,'fold_mean'] = df_results.mean(numeric_only=True, axis=1)\ndf_results","metadata":{"papermill":{"duration":0.254835,"end_time":"2022-01-02T19:39:47.079815","exception":false,"start_time":"2022-01-02T19:39:46.82498","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T06:46:30.591464Z","iopub.execute_input":"2022-07-31T06:46:30.592580Z","iopub.status.idle":"2022-07-31T06:46:30.618889Z","shell.execute_reply.started":"2022-07-31T06:46:30.592531Z","shell.execute_reply":"2022-07-31T06:46:30.617894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_importance(importances, features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()\n    \nplot_importance(np.array(importances),train_data.columns, PLOT_TOP_N = 20, figsize=(10, 20))","metadata":{"papermill":{"duration":0.229805,"end_time":"2022-01-02T19:39:48.026186","exception":false,"start_time":"2022-01-02T19:39:47.796381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-31T06:46:30.620576Z","iopub.execute_input":"2022-07-31T06:46:30.621044Z","iopub.status.idle":"2022-07-31T06:46:31.628282Z","shell.execute_reply.started":"2022-07-31T06:46:30.620998Z","shell.execute_reply":"2022-07-31T06:46:31.627302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importance_split_df = pd.DataFrame(data=importances_split, columns=train_data.columns)\nsorted_split = importance_split_df.median(axis=0).sort_values(ascending=False)\nsorted_split.head()\n\nprint(f'nb features used:{len(sorted_split[sorted_split>1])}')","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:51:50.452916Z","iopub.execute_input":"2022-07-31T06:51:50.453382Z","iopub.status.idle":"2022-07-31T06:51:50.528863Z","shell.execute_reply.started":"2022-07-31T06:51:50.453343Z","shell.execute_reply":"2022-07-31T06:51:50.527684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_importance_groups(importances, features_names = train_data.columns, PLOT_TOP_N = 20, figsize=(4, 8)):\n    importance_df = pd.DataFrame(data=importances, columns=features_names)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    \n    t = sorted_importance_df.transpose()\n    t['groups'] = [s.split('_')[-1] for s in sorted_importance_df.columns]\n    \n    t = t.groupby('groups').sum().transpose()\n    t = t.loc[:,t.columns.isin(['last','max','mean','min','std'])]\n\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    #ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=t,\n                orient='h',\n                ax=ax)\n    plt.show()\n    \nplot_importance_groups(np.array(importances))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.819122Z","iopub.status.idle":"2022-07-31T06:46:31.819557Z","shell.execute_reply.started":"2022-07-31T06:46:31.819358Z","shell.execute_reply":"2022-07-31T06:46:31.819378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"indeed 'last' features play a major role. ","metadata":{}},{"cell_type":"code","source":"del train_data, train_labels, X_tr, X_va, y_tr, y_va ","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.821001Z","iopub.status.idle":"2022-07-31T06:46:31.821446Z","shell.execute_reply.started":"2022-07-31T06:46:31.821255Z","shell.execute_reply":"2022-07-31T06:46:31.821277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission\n","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_pickle('../input/amex-feature-engineering/test_data_agg.pkl').astype('float16')\n\nmissing_cols = [f for f in Features if f not in test_data.columns]\ntest_data[missing_cols] = 0","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.823036Z","iopub.status.idle":"2022-07-31T06:46:31.823451Z","shell.execute_reply.started":"2022-07-31T06:46:31.823266Z","shell.execute_reply":"2022-07-31T06:46:31.823287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/2130016/splitting-a-list-into-n-parts-of-approximately-equal-length\ndef split(a, n):\n    k, m = divmod(len(a), n)\n    return (a[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n))\n\nsplit_ids = split(test_data.index.unique(),10)\n\ndf_list_preds = []","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.824970Z","iopub.status.idle":"2022-07-31T06:46:31.825398Z","shell.execute_reply.started":"2022-07-31T06:46:31.825210Z","shell.execute_reply":"2022-07-31T06:46:31.825231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_sub = []\n\nfor (i,ids) in enumerate(split_ids):\n    print(f'chunk {i}')\n    test_data_ids = test_data[test_data.index.isin(ids)][Features]\n    preds_ids_sub = []\n    for k in models:\n        print(f'model key {k}')\n        preds_ids_sub.append(models[k].predict(test_data_ids, raw_score=True))\n    preds_sub.append(np.nanmean(np.array(preds_ids_sub),axis=0))\n    gc.collect()\n    \npreds_sub = np.hstack(preds_sub)\npreds_series = pd.Series(preds_sub,index = test_data.index.unique())\nproba_series = np.exp(preds_series)/(1+np.exp(preds_series))","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.826657Z","iopub.status.idle":"2022-07-31T06:46:31.827101Z","shell.execute_reply.started":"2022-07-31T06:46:31.826874Z","shell.execute_reply":"2022-07-31T06:46:31.826894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds_series.hist(bins=100)\n# plt.axvline(x=np.log(0.04/(1-0.04)),color='black');","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.828279Z","iopub.status.idle":"2022-07-31T06:46:31.828653Z","shell.execute_reply.started":"2022-07-31T06:46:31.828482Z","shell.execute_reply":"2022-07-31T06:46:31.828500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# proba_series.hist(bins=100)\n# plt.axvline(x=0.04,color='black');","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.829746Z","iopub.status.idle":"2022-07-31T06:46:31.830160Z","shell.execute_reply.started":"2022-07-31T06:46:31.829960Z","shell.execute_reply":"2022-07-31T06:46:31.829985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\ndf_sub.prediction = proba_series.loc[df_sub.customer_ID].values\n\ndf_sub = df_sub.set_index('customer_ID')\ndf_sub.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-31T06:46:31.831136Z","iopub.status.idle":"2022-07-31T06:46:31.831511Z","shell.execute_reply.started":"2022-07-31T06:46:31.831338Z","shell.execute_reply":"2022-07-31T06:46:31.831356Z"},"trusted":true},"execution_count":null,"outputs":[]}]}