{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style=\"color: #f8f8ff;\n           display:fill;\n           border-radius:10px;\n           border-style: solid;\n           border-color:#424949;\n           text-align:center;\n           background-color:#69541b ;\n           font-size:20px;\n           letter-spacing:0.5px;\n           padding: 0.7em;\n           text-align:left\">  \n<center>  Background  </center> \n<head> \n\n</head> \n <hr>\n <ul>\n     The content of this notebook mainly comes from <a href=\"https://www.kaggle.com/code/ogrellier/feature-selection-with-null-importances\" color: red>here.</a> Thanks to @olivier for his contribution\n <hr>\n The notebook implements the following steps :\n\n<li>Create the null importances distributions : these are created fitting the model over several runs on a shuffled version of the target. This shows how the model can make sense of a feature irrespective of the target.\n<li>Fit the model on the original target and gather the feature importances. This gives us a benchmark whose significance can be tested against the Null Importances Distribution\n<li>for each feature test the actual importance:\n<li>Compute the probabability of the actual importance wrt the null distribution. I will use a very simple estimation using occurences while the article proposes to fit known distribution to the gathered data. In fact here I'll compute 1 - the proba so that things are in the right order.\n<li>Simply compare the actual importance to the mean and max of the null importances. This will give sort of a feature importance that allows to see major features in the dataset. Indeed the previous method may give us lots of ones.\n </ul>\n <hr>\n</div>","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport eli5\nimport lightgbm as lgb\nimport time\nimport gc\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nimport warnings\nwarnings.simplefilter('ignore', UserWarning)\ngc.enable()\n\nfrom typing import Dict, Tuple, List, Union\nfrom pandas import DataFrame, Series\nfrom contextlib import contextmanager\nfrom sklearn.model_selection import  train_test_split\nfrom sklearn.metrics import accuracy_score, recall_score, precision_score\nfrom eli5.sklearn import PermutationImportance","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:10:12.051334Z","iopub.execute_input":"2022-06-18T04:10:12.051818Z","iopub.status.idle":"2022-06-18T04:10:21.164498Z","shell.execute_reply.started":"2022-06-18T04:10:12.051737Z","shell.execute_reply":"2022-06-18T04:10:21.163657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"color: #fff7f7;\n           display:fill;\n           border-radius:10px;\n           border-style: solid;\n           border-color:#424949;\n           text-align:center;\n           background-color:#69541b ;\n           font-size:20px;\n           letter-spacing:0.5px;\n           padding: 0.7em;\n           text-align:left\">  \n<center>  Seed and load data </center> \n</div>","metadata":{}},{"cell_type":"code","source":"SEED=2022\ndef seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n\n@contextmanager\ndef timer(name: str):\n    s = time.time()\n    yield\n    S_time = time.time() - s\n    print(f'[{name}] {S_time: .2f}sec')\n\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:10:21.166851Z","iopub.execute_input":"2022-06-18T04:10:21.168083Z","iopub.status.idle":"2022-06-18T04:10:21.174574Z","shell.execute_reply.started":"2022-06-18T04:10:21.16804Z","shell.execute_reply":"2022-06-18T04:10:21.17386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_feather(\"../input/amexfeather/train_data.ftr\")","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:10:21.176447Z","iopub.execute_input":"2022-06-18T04:10:21.177447Z","iopub.status.idle":"2022-06-18T04:10:41.736315Z","shell.execute_reply.started":"2022-06-18T04:10:21.177394Z","shell.execute_reply":"2022-06-18T04:10:41.735351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"color: #fff7f7;\n           display:fill;\n           border-radius:10px;\n           border-style: solid;\n           border-color:#424949;\n           text-align:center;\n           background-color:#69541b ;\n           font-size:20px;\n           letter-spacing:0.5px;\n           padding: 0.7em;\n           text-align:left\">  \n<center>  Num-feature and Cat-feature </center> \n</div>","metadata":{}},{"cell_type":"code","source":"cols = train.columns.to_list()\ncategory_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nnumerical_cols = [col for col in cols if col not in category_cols + ['target',\"S_2\",\"customer_ID\"]]\nall_cols = category_cols + numerical_cols\n\n\nfor cat in category_cols:\n    train[cat] = pd.factorize(train[cat])[0]\n    train[cat] = train[cat].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:10:41.738186Z","iopub.execute_input":"2022-06-18T04:10:41.738512Z","iopub.status.idle":"2022-06-18T04:10:43.459991Z","shell.execute_reply.started":"2022-06-18T04:10:41.738478Z","shell.execute_reply":"2022-06-18T04:10:43.459175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Split data, also reduce data\ntrain_x , valid_x, train_y, valid_y = train_test_split(train[all_cols], train[[\"target\"]], test_size=0.35, stratify=train[\"target\"])  \ntrain = pd.concat([train_x, train_y], axis=1)\ntrain = train.reset_index(drop=True)\n\n\ndel train_x, train_y, valid_x, valid_y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:10:43.461334Z","iopub.execute_input":"2022-06-18T04:10:43.461813Z","iopub.status.idle":"2022-06-18T04:11:07.810114Z","shell.execute_reply.started":"2022-06-18T04:10:43.461777Z","shell.execute_reply":"2022-06-18T04:11:07.809309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"color: #fff7f7;\n           display:fill;\n           border-radius:10px;\n           border-style: solid;\n           border-color:#424949;\n           text-align:center;\n           background-color:#69541b ;\n           font-size:20px;\n           letter-spacing:0.5px;\n           padding: 0.7em;\n           text-align:left\">  \n<center>  Train lgbm-model </center> ","metadata":{}},{"cell_type":"code","source":"def get_feature_importance_lgbm(data:DataFrame, shuffle:bool = False, seed:int = 2022) -> Tuple:\n    \n    \n    \"\"\"\n    args:\n        data: pd.DataFrame\n        shuffle: default=False,Used to randomly shuffle the target value.\n        seed:random seed for the lgbm-model\n        \n    returns:\n        Tuple: \n          Tuple[0](model object):trained model\n          Tuple[1](DataFrame):Feature importance after training\n    \n    \"\"\"\n    \n    all_features = [f for f in data if f not in ['target', 'S_2', \"customer_ID\"]]\n    \n    #Take random scrambled label data and get the scrambled feature importances. \n    #if shuffle=True  These feature importances are wrong.\n    y = data['target'].copy()\n    if shuffle:\n        y = data['target'].copy().sample(frac=1.0)\n        \n    lgb_params = {\n        'boosting_type': 'rf',\n        'subsample': 0.6,\n        'colsample_bytree': 0.6,\n        'num_leaves': 200,\n        'max_depth': 10,\n        'seed ': seed,\n        'bagging_freq': 1,\n        \"n_jobs\":4,\n        \"bagging_seed \":seed,\n        \"min_gain_to_split\":0.10\n    }\n    \n    clf = lgb.LGBMClassifier(**lgb_params)\n    clf.fit(data[all_features], y, categorical_feature=category_cols)\n\n    imp_df = pd.DataFrame()\n    imp_df[\"feature\"] = list(all_features)\n    imp_df[\"importance_gain\"] = clf.booster_.feature_importance(importance_type='gain')\n    imp_df[\"importance_split\"] = clf.booster_.feature_importance(importance_type='split')\n    imp_df['train_accuracy'] = accuracy_score(y, clf.predict(data[all_features]))\n    imp_df['train_recall'] = recall_score(y, clf.predict(data[all_features]))\n    imp_df['train_precision'] = precision_score(y, clf.predict(data[all_features]))\n    \n    return clf, imp_df","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:11:07.81167Z","iopub.execute_input":"2022-06-18T04:11:07.812108Z","iopub.status.idle":"2022-06-18T04:11:07.822658Z","shell.execute_reply.started":"2022-06-18T04:11:07.812065Z","shell.execute_reply":"2022-06-18T04:11:07.821804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with timer(\"train model:\"):\n    model, truly_imp_df = get_feature_importance_lgbm(data=train)\n    truly_imp_df.to_csv(\"truly_feature_importance.csv\")  #Save the data for easy comparison of filtered features in another notebook.","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:11:07.824199Z","iopub.execute_input":"2022-06-18T04:11:07.824996Z","iopub.status.idle":"2022-06-18T04:18:41.307985Z","shell.execute_reply.started":"2022-06-18T04:11:07.824954Z","shell.execute_reply":"2022-06-18T04:18:41.307115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"truly_imp_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:18:41.309174Z","iopub.execute_input":"2022-06-18T04:18:41.310193Z","iopub.status.idle":"2022-06-18T04:18:41.326052Z","shell.execute_reply.started":"2022-06-18T04:18:41.310153Z","shell.execute_reply":"2022-06-18T04:18:41.32523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"color: #fff7f7;\n           display:fill;\n           border-radius:10px;\n           border-style: solid;\n           border-color:#424949;\n           text-align:center;\n           background-color:#69541b ;\n           font-size:20px;\n           letter-spacing:0.5px;\n           padding: 0.7em;\n           text-align:left\">  \n<center>  Explain feature importances - wight </center> \n</div>","metadata":{}},{"cell_type":"code","source":"eli5.show_weights(model, feature_names = all_cols, importance_type=\"split\", top=20)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:18:41.327297Z","iopub.execute_input":"2022-06-18T04:18:41.328085Z","iopub.status.idle":"2022-06-18T04:18:41.364885Z","shell.execute_reply.started":"2022-06-18T04:18:41.328048Z","shell.execute_reply":"2022-06-18T04:18:41.363873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eli5.show_weights(model, feature_names = all_cols, importance_type=\"gain\", top=20)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:18:41.368383Z","iopub.execute_input":"2022-06-18T04:18:41.368915Z","iopub.status.idle":"2022-06-18T04:18:41.379019Z","shell.execute_reply.started":"2022-06-18T04:18:41.368885Z","shell.execute_reply":"2022-06-18T04:18:41.378114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div style=\"color: #fff7f7;\n           display:fill;\n           border-radius:10px;\n           border-style: solid;\n           border-color:#424949;\n           text-align:center;\n           background-color:#69541b ;\n           font-size:20px;\n           letter-spacing:0.5px;\n           padding: 0.7em;\n           text-align:left\">  \n<center>  Null feature importance </center>\n</div>","metadata":{}},{"cell_type":"code","source":"with timer(\"build null feature importance\"):\n    \n    null_imp = pd.DataFrame()\n    runs = 5\n    for i in range(runs):\n        model, imp_df = get_feature_importance_lgbm(data=train, shuffle=True) # return (model, df) \n        imp_df[\"run_num\"] = i+1\n        null_imp = pd.concat([null_imp, imp_df], axis=0)\n        \n        del model\n        gc.collect()\n        print(f\"======runing:{i+1}======\")\n        \n    null_imp.to_csv(\"null_feature_importance_with_5.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:18:41.380597Z","iopub.execute_input":"2022-06-18T04:18:41.381174Z","iopub.status.idle":"2022-06-18T04:46:20.333212Z","shell.execute_reply.started":"2022-06-18T04:18:41.381135Z","shell.execute_reply":"2022-06-18T04:46:20.332392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_imp.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:46:20.334358Z","iopub.execute_input":"2022-06-18T04:46:20.335715Z","iopub.status.idle":"2022-06-18T04:46:20.351743Z","shell.execute_reply.started":"2022-06-18T04:46:20.335662Z","shell.execute_reply":"2022-06-18T04:46:20.350898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_distributions(actual_imp_df_:DataFrame, null_imp_df_:DataFrame, feature:str) -> None:\n    \n    \"\"\"\n    args:\n      actual_imp_df_:Unshuffled data\n      null_imp_df_：Data that has been shuffled many times. At present, it has gone through 5 shuffles\n      feature:Feature columns in the data.\n      \n    \"\"\"\n    \n    plt.figure(figsize=(13, 6))\n    gs = gridspec.GridSpec(1, 2)\n    \n    # 1、Plot Split importances\n    ax = plt.subplot(gs[0, 0])\n    a = ax.hist(null_imp_df_.loc[null_imp_df_['feature'] == feature, 'importance_split'].values, label='Null importances')\n    ax.vlines(x=actual_imp_df_.loc[actual_imp_df_['feature'] == feature, 'importance_split'].mean(), \n               ymin=0, ymax=np.max(a[0]), color='r',linewidth=10, label='Real Target')\n    ax.legend()\n    ax.set_title('Split Importance of %s' % feature.upper(), fontweight='bold')\n    ax.spines[['top', 'right']].set_visible(False)\n    ax.spines[['left','bottom']].set_linewidth(1.5)\n    ax.grid(False)\n    plt.xlabel('Null Importance (split) Distribution for %s ' % feature.upper())\n    \n    # 2、Plot Gain importances\n    ax = plt.subplot(gs[0, 1])\n    a = ax.hist(null_imp_df_.loc[null_imp_df_['feature'] == feature, 'importance_gain'].values, label='Null importances')\n    ax.vlines(x=actual_imp_df_.loc[actual_imp_df_['feature'] == feature, 'importance_gain'].mean(), \n               ymin=0, ymax=np.max(a[0]), color='r',linewidth=10, label='Real Target')\n    ax.legend()\n    ax.set_title('Gain Importance of %s' % feature.upper(), fontweight='bold')\n    ax.spines[['top', 'right']].set_visible(False)\n    ax.spines[['left','bottom']].set_linewidth(1.5)\n    ax.grid(False)\n    plt.xlabel('Null Importance (gain) Distribution for %s ' % feature.upper())","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:46:20.352891Z","iopub.execute_input":"2022-06-18T04:46:20.353245Z","iopub.status.idle":"2022-06-18T04:46:20.366882Z","shell.execute_reply.started":"2022-06-18T04:46:20.353197Z","shell.execute_reply":"2022-06-18T04:46:20.365728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- View the importance of the original features at the top, and the distribution graph after shuffle.","metadata":{}},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"P_2\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:20.368376Z","iopub.execute_input":"2022-06-18T04:46:20.368813Z","iopub.status.idle":"2022-06-18T04:46:20.756951Z","shell.execute_reply.started":"2022-06-18T04:46:20.368775Z","shell.execute_reply":"2022-06-18T04:46:20.754756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"D_42\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:20.758277Z","iopub.execute_input":"2022-06-18T04:46:20.758659Z","iopub.status.idle":"2022-06-18T04:46:21.106923Z","shell.execute_reply.started":"2022-06-18T04:46:20.758622Z","shell.execute_reply":"2022-06-18T04:46:21.105099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"B_9\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:21.108121Z","iopub.execute_input":"2022-06-18T04:46:21.108584Z","iopub.status.idle":"2022-06-18T04:46:21.463076Z","shell.execute_reply.started":"2022-06-18T04:46:21.108543Z","shell.execute_reply":"2022-06-18T04:46:21.462265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"S_3\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:21.464353Z","iopub.execute_input":"2022-06-18T04:46:21.465012Z","iopub.status.idle":"2022-06-18T04:46:21.818168Z","shell.execute_reply.started":"2022-06-18T04:46:21.464969Z","shell.execute_reply":"2022-06-18T04:46:21.817305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"B_3\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:21.819366Z","iopub.execute_input":"2022-06-18T04:46:21.819836Z","iopub.status.idle":"2022-06-18T04:46:22.167119Z","shell.execute_reply.started":"2022-06-18T04:46:21.819797Z","shell.execute_reply":"2022-06-18T04:46:22.166343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"D_48\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:22.168565Z","iopub.execute_input":"2022-06-18T04:46:22.16908Z","iopub.status.idle":"2022-06-18T04:46:22.510689Z","shell.execute_reply.started":"2022-06-18T04:46:22.169039Z","shell.execute_reply":"2022-06-18T04:46:22.509877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"D_63\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:22.511816Z","iopub.execute_input":"2022-06-18T04:46:22.51264Z","iopub.status.idle":"2022-06-18T04:46:22.838164Z","shell.execute_reply.started":"2022-06-18T04:46:22.512598Z","shell.execute_reply":"2022-06-18T04:46:22.837362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"D_64\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-18T04:46:22.839363Z","iopub.execute_input":"2022-06-18T04:46:22.839864Z","iopub.status.idle":"2022-06-18T04:46:23.198743Z","shell.execute_reply.started":"2022-06-18T04:46:22.839823Z","shell.execute_reply":"2022-06-18T04:46:23.197954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"D_66\")","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_distributions(truly_imp_df, null_imp, feature=\"D_68\")","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above plot I believe the power of the exposed feature selection method is demonstrated. In particular it is well known that :\n\nAny feature sufficient variance can be used and made sense of by tree models. You can always find splits that help scoring better\nCorrelated features have decaying importances once one of them is used by the model. The chosen feature will have strong importance and its correlated suite will have decaying importances\n\nThe current method allows to :\n\nDrop high variance features if they are not really related to the target\nRemove the decaying factor on correlated features, showing their real importance (or unbiased importance)","metadata":{}},{"cell_type":"markdown","source":"#### Score features <br>\n<h>  There are several ways to score features :\n\n- Compute the number of samples in the actual importances that are away from the null importances recorded distribution.\n- Compute ratios like Actual / Null Max, Actual / Null Mean, Actual Mean / Null Max <br>\nIn a first step I will use the log actual feature importance divided by the 75 percentile of null distribution.","metadata":{}},{"cell_type":"code","source":"def plot_feature_scores(true_df:DataFrame, null_df:DataFrame) -> None:\n    \n    def score_df():\n        \n        feature_scores = []\n        for f in true_df['feature'].unique():\n            f_null_imps_gain = null_df.loc[null_df['feature'] == f, 'importance_gain'].values\n            f_act_imps_gain = true_df.loc[true_df['feature'] == f, 'importance_gain'].mean()\n            gain_score = np.log(1e-10 + f_act_imps_gain / (1 + np.percentile(f_null_imps_gain, 75)))     # Avoid didvide by zero\n            f_null_imps_split = null_df.loc[null_df['feature'] == f, 'importance_split'].values\n            f_act_imps_split = true_df.loc[true_df['feature'] == f, 'importance_split'].mean()\n            split_score = np.log(1e-10 + f_act_imps_split / (1 + np.percentile(f_null_imps_split, 75)))  # Avoid didvide by zero\n            feature_scores.append((f, split_score, gain_score))\n            \n        scores_df = pd.DataFrame(feature_scores, columns=['feature', 'split_score', 'gain_score'])\n        \n        scores_df.to_csv(\"score_feature_with_mean.csv\")\n        return scores_df\n    \n    \n    plt.figure(figsize=(16, 16))\n    gs = gridspec.GridSpec(1, 2)\n    \n    # 1、Plot Split importances\n    ax = plt.subplot(gs[0, 0])\n    sns.barplot(x='split_score', y='feature', data=score_df().sort_values('split_score', ascending=False).iloc[0:70], ax=ax)\n    ax.set_title('Feature scores wrt split importances', fontweight='bold', fontsize=14)\n    ax.spines[['top', 'right']].set_visible(False)\n    ax.spines[['left','bottom']].set_linewidth(1.5)\n    ax.grid(False)\n    \n    # 2、Plot Gain importances\n    ax = plt.subplot(gs[0, 1])\n    sns.barplot(x='gain_score', y='feature', data=score_df().sort_values('gain_score', ascending=False).iloc[0:70], ax=ax)\n    ax.set_title('Feature scores wrt gain importances', fontweight='bold', fontsize=14)\n    ax.spines[['top', 'right']].set_visible(False)\n    ax.spines[['left','bottom']].set_linewidth(1.5)\n    ax.grid(False)\n    plt.tight_layout()\n        \n\n#=======================================\nplot_feature_scores(truly_imp_df, null_imp)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:46:23.200099Z","iopub.execute_input":"2022-06-18T04:46:23.200692Z","iopub.status.idle":"2022-06-18T04:46:23.727465Z","shell.execute_reply.started":"2022-06-18T04:46:23.200649Z","shell.execute_reply":"2022-06-18T04:46:23.72631Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check the impact of removing uncorrelated features <br>\n- use a different metric to asses correlation to the target","metadata":{}},{"cell_type":"code","source":"def plot_corr_scores(true_df:DataFrame, null_df:DataFrame) -> None:\n    \n    \n    correlation_scores = []\n    def corr_score_df():\n        \n        \n        for f in true_df['feature'].unique():\n            f_null_imps = null_df.loc[null_df['feature'] == f, 'importance_gain'].values\n            f_act_imps = true_df.loc[true_df['feature'] == f, 'importance_gain'].values\n            gain_score = 100 * (f_null_imps < np.percentile(f_act_imps, 25)).sum() / f_null_imps.size\n            f_null_imps = null_df.loc[null_df['feature'] == f, 'importance_split'].values\n            f_act_imps = true_df.loc[true_df['feature'] == f, 'importance_split'].values\n            split_score = 100 * (f_null_imps < np.percentile(f_act_imps, 25)).sum() / f_null_imps.size\n            correlation_scores.append((f, split_score, gain_score))\n\n        corr_scores_df = pd.DataFrame(correlation_scores, columns=['feature', 'split_score', 'gain_score'])\n        corr_scores_df.to_csv(\"corr_score_filter.csv\")\n        \n        return correlation_scores, corr_scores_df\n    \n    \n    correlation_scores, corr_scores_df = corr_score_df()\n    \n    fig = plt.figure(figsize=(16, 16))\n    gs = gridspec.GridSpec(1, 2)\n\n    # Plot Split importances\n    ax = plt.subplot(gs[0, 0])\n    sns.barplot(x='split_score', y='feature', data=corr_scores_df.sort_values('split_score', ascending=False).iloc[0:70], ax=ax)\n    ax.set_title('Feature scores wrt split importances', fontweight='bold', fontsize=14)\n    ax.spines[['top', 'right']].set_visible(False)\n    ax.spines[['left','bottom']].set_linewidth(1.5)\n    ax.grid(False)\n\n    # Plot Gain importances\n    ax = plt.subplot(gs[0, 1])\n    sns.barplot(x='gain_score', y='feature', data=corr_scores_df.sort_values('gain_score', ascending=False).iloc[0:70], ax=ax)\n    ax.set_title('Feature scores wrt gain importances', fontweight='bold', fontsize=14)\n    ax.spines[['top', 'right']].set_visible(False)\n    ax.spines[['left','bottom']].set_linewidth(1.5)\n    ax.grid(False)\n    plt.tight_layout()\n    plt.suptitle(\"Features' split and gain scores\", fontweight='bold', fontsize=16)\n    fig.subplots_adjust(top=0.93)\n    \n    gc.collect()\n    \n    return correlation_scores\n\n\n#=====================\ncorrelation_scores = plot_corr_scores(truly_imp_df, null_imp)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:46:23.728838Z","iopub.status.idle":"2022-06-18T04:46:23.729541Z","shell.execute_reply.started":"2022-06-18T04:46:23.729286Z","shell.execute_reply":"2022-06-18T04:46:23.729311Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def com_feature(a:List, b:List) -> List:\n    \n    #get the same element in two lists\n    \n    return set(a) & set(b)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:46:23.730664Z","iopub.status.idle":"2022-06-18T04:46:23.731517Z","shell.execute_reply.started":"2022-06-18T04:46:23.73126Z","shell.execute_reply":"2022-06-18T04:46:23.731285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Score feature removal for different thresholds :)","metadata":{}},{"cell_type":"code","source":"def score_feature_selection(df=None, train_features=None, cat_feats=None, target=None):\n\n    dtrain = lgb.Dataset(df[train_features], target, free_raw_data=False, silent=True)\n    lgb_params = {\n        'objective': 'binary',\n        'boosting_type': 'gbdt',\n        'learning_rate': .1,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'num_leaves': 31,\n        'max_depth': -1,\n        'seed': 13,\n        'n_jobs': 4,\n        'min_split_gain': .01,\n        'reg_alpha': .0001,\n        'reg_lambda': .0001,\n        'metric': 'auc'\n    }\n    \n    # Fit the model\n    hist = lgb.cv(\n        params=lgb_params, \n        train_set=dtrain, \n        num_boost_round=300,\n        categorical_feature=cat_feats,\n        nfold=5, \n        stratified=True,\n        shuffle=True,\n        early_stopping_rounds=20,\n        verbose_eval=0,\n        seed=17\n    )\n    \n    # Return the last mean / std values \n    return hist['auc-mean'][-1], hist['auc-stdv'][-1]\n\n\nfor threshold in [10, 30, 50, 70, 90, 99]:\n    split_feats = [f for f, score, _ in correlation_scores if score >= threshold]\n    split_cat_feats = [f for f, score, _ in correlation_scores if (score >= threshold) & (f in category_cols)]\n    gain_feats = [f for f, _, score in correlation_scores if score >= threshold]\n    gain_cat_feats = [f for f, _, score in correlation_scores if (score >= threshold) & (f in category_cols)]\n    \n    \n    print('Results for threshold %3d' % threshold)\n    print(\"The selected features are now：\")\n    \n    \n    print(\"[the same element:]{}\".format(com_feature(split_feats, gain_feats)))\n    print(\"[the same cat_element]:{}.\".format(com_feature(split_cat_feats, gain_cat_feats)))\n\n    split_results = score_feature_selection(df=train, train_features=split_feats, cat_feats=split_cat_feats, target=train['target'])\n    print('\\t SPLIT : %.6f +/- %.6f' % (split_results[0], split_results[1]))\n    \n    gain_results = score_feature_selection(df=train, train_features=gain_feats, cat_feats=gain_cat_feats, target=train['target'])\n    print('\\t GAIN  : %.6f +/- %.6f' % (gain_results[0], gain_results[1]))\n    \n    print(\"==================\")\n    del split_feats, split_cat_feats, gain_feats, gain_cat_feats\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-18T04:46:23.732786Z","iopub.status.idle":"2022-06-18T04:46:23.733635Z","shell.execute_reply.started":"2022-06-18T04:46:23.733386Z","shell.execute_reply":"2022-06-18T04:46:23.733411Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Due to memory reasons, the next step will try to model and analyze the obtained feature importance data file in another notebook, and compare the effect of the filtered model.   ：)","metadata":{}}]}