{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Simple weight of evidence baseline; WoE is a target encoding technique replacing values by an associated value that has nice additive properties.\nGive strong baseline, generally at the cost of feature interactions.\n\n**Don't Forget to upvote if you find this interesting or usefull**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\n# matplotlib setting\nmpl.rcParams['figure.dpi'] = 200\nmpl.rcParams['axes.spines.top'] = False\nmpl.rcParams['axes.spines.right'] = False\n\n# pandas setting\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 1000)\n\nDEBUG = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-27T14:01:22.729549Z","iopub.execute_input":"2022-05-27T14:01:22.729827Z","iopub.status.idle":"2022-05-27T14:01:23.849726Z","shell.execute_reply.started":"2022-05-27T14:01:22.729798Z","shell.execute_reply":"2022-05-27T14:01:23.848477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate_data","metadata":{}},{"cell_type":"code","source":"%%time\ntrain_data_grp = pd.read_pickle('../input/amex-feature-engineering/train_data_agg.pkl')\ntrain_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv').set_index('customer_ID').loc[train_data_grp.index]","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:01:23.851593Z","iopub.execute_input":"2022-05-27T14:01:23.851886Z","iopub.status.idle":"2022-05-27T14:01:33.910755Z","shell.execute_reply.started":"2022-05-27T14:01:23.851840Z","shell.execute_reply":"2022-05-27T14:01:33.909542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_data_grp = pd.read_pickle('../input/amex-feature-engineering/test_data_agg.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:01:33.912331Z","iopub.execute_input":"2022-05-27T14:01:33.912579Z","iopub.status.idle":"2022-05-27T14:01:51.915456Z","shell.execute_reply.started":"2022-05-27T14:01:33.912550Z","shell.execute_reply":"2022-05-27T14:01:51.914194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# weight of evidence\n\nStandard Credit Scoring Technique. Target encoding technique that replace feature value by an additive value that helps build credit Scorecards. Personal sklearn implementation (doesn't handle edge case very well).","metadata":{}},{"cell_type":"code","source":"class WoE_Imputer(BaseEstimator, TransformerMixin):\n# Bins the features and impute Weight of Evidence associated with each bin\n# Weight of Evidence is calculated as the log ratio of positive outcome to negative ones in each bin\n# This imputation technique is adapted to the specific functionnal form of logistic regression\n# Allows to impute missing values\n# Also allows to calculate Information Values for feature selection\n    def __init__(self, feature_name, n_bin = 100, Categorical = False, verbosity = 1):  \n        self.feature_name = feature_name\n        self.n_bin = n_bin\n        self.bins = []\n        self.WoE_values = []\n        self.Categorical = Categorical \n        self.verbosity = verbosity\n        self.IV = 0\n\n    def fit(self, X, y = None):\n        if y is None:\n            raise ValueError('Woe Imputer is a supervised imputer. It needs a target')\n\n        if self.Categorical:\n            values_quantiles = X[self.feature_name].astype('category')\n            self.bins = values_quantiles.cat\n        else:\n            values_quantiles, self.bins = pd.qcut(X[self.feature_name], q=self.n_bin, duplicates = 'drop', retbins=True)   \n            self.bins[0] = -np.Inf\n            self.bins[-1] = np.Inf\n            values_quantiles = pd.cut(X[self.feature_name], bins = self.bins)\n\n        values_quantiles = values_quantiles.cat.add_categories('missing_value')\n        values_quantiles.fillna('missing_value', inplace = True) \n\n        df = pd.DataFrame({'group': values_quantiles, 'val': X[self.feature_name], 'target': y.values.flatten()})\n\n        sum_positive_by_quantile = df.groupby('group').sum().target\n        sum_negative_by_quantile = df.groupby('group').count().target - df.groupby('group').sum().target\n\n        data = np.log(sum_positive_by_quantile / sum_negative_by_quantile)\n        \n        #interpolate in case of na - there are other tricks\n        mask = np.isnan(data)\n        data[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), data[~mask])\n\n        self.WoE_values =  data\n\n        self.IV = ((sum_positive_by_quantile - sum_negative_by_quantile) * self.WoE_values / df.shape[0]).sum()\n\n        if self.verbosity>0:\n            print('Information Value ' + str(self.feature_name)+': ' + str(round(self.IV,5)))\n            \n        return self\n\n    def transform(self, X):\n        feature_to_transform = X[self.feature_name].copy()\n        transformed_feature = pd.cut(feature_to_transform, bins =  self.bins, labels = np.array(self.WoE_values[:-1]), ordered = False).astype('float32')\n        transformed_feature = transformed_feature.replace(np.nan, self.WoE_values[-1])\n        X[self.feature_name] = transformed_feature\n        return X\n\n    def __get_val__(self):  \n        return self.feature_name, self.n_bin, self.bins, self.WoE_values, self.IV","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:01:51.919295Z","iopub.execute_input":"2022-05-27T14:01:51.919594Z","iopub.status.idle":"2022-05-27T14:01:51.941094Z","shell.execute_reply.started":"2022-05-27T14:01:51.919553Z","shell.execute_reply":"2022-05-27T14:01:51.940209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm_notebook\n\nFeatures = train_data_grp.columns\nFeatures = [f for f in Features if not f.startswith('target')]\n\nif DEBUG:\n    Features = Features[:20]\n\nIV_list = []\nFeatures_list = []\n\nFeatures_list_not_ok = []\n\nfor f in tqdm_notebook(Features):\n    try:\n        WoE_imp = WoE_Imputer(f, n_bin = 50, verbosity = 0)\n        WoE_imp.fit(train_data_grp, y = train_labels.target)\n        train_data_grp = WoE_imp.transform(train_data_grp)\n        test_data_grp = WoE_imp.transform(test_data_grp)\n        feature_name, n_bin, bins, WoE_values, IV = WoE_imp.__get_val__()\n        Features_list.append(feature_name)\n        IV_list.append(IV)\n    except:\n        Features_list_not_ok.append(f)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:01:51.942337Z","iopub.execute_input":"2022-05-27T14:01:51.942561Z","iopub.status.idle":"2022-05-27T14:06:16.783670Z","shell.execute_reply.started":"2022-05-27T14:01:51.942533Z","shell.execute_reply":"2022-05-27T14:06:16.782463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_IV = pd.DataFrame({'Features':Features_list,'IV':IV_list}).sort_values('IV',ascending=False).reset_index(drop=True)\nplt.plot(sorted_IV.IV);","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:06:16.784923Z","iopub.execute_input":"2022-05-27T14:06:16.785191Z","iopub.status.idle":"2022-05-27T14:06:17.079022Z","shell.execute_reply.started":"2022-05-27T14:06:16.785159Z","shell.execute_reply":"2022-05-27T14:06:17.077974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inf_value = sorted_IV.IV[0]\n\nsorted_IV = sorted_IV[sorted_IV.IV != inf_value]","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:09:22.573100Z","iopub.execute_input":"2022-05-27T14:09:22.573409Z","iopub.status.idle":"2022-05-27T14:09:22.581712Z","shell.execute_reply.started":"2022-05-27T14:09:22.573380Z","shell.execute_reply":"2022-05-27T14:09:22.580710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:06:17.088572Z","iopub.execute_input":"2022-05-27T14:06:17.088858Z","iopub.status.idle":"2022-05-27T14:06:17.104972Z","shell.execute_reply.started":"2022-05-27T14:06:17.088821Z","shell.execute_reply":"2022-05-27T14:06:17.104269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"pos / (pos + neg) = 1/(1+neg/pos) = (1/1+exp(-log(pos/neg)))","metadata":{}},{"cell_type":"code","source":"top_n = 625\n\nlist_features = sorted_IV.Features[:top_n].to_list()\npred_train = train_data_grp[list_features].mean(axis=1)\nprob_train = 1/(1+np.exp(-pred_train))\nmetric = amex_metric(train_labels, prob_train.rename('prediction'))\nprint(f'n {top_n} - {metric:.2%}')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:17:20.193599Z","iopub.execute_input":"2022-05-27T14:17:20.193919Z","iopub.status.idle":"2022-05-27T14:19:33.660671Z","shell.execute_reply.started":"2022-05-27T14:17:20.193889Z","shell.execute_reply":"2022-05-27T14:19:33.659746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"df_sub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\n\npred_test = test_data_grp[list_features].mean(axis=1)\nprob_test = 1/(1+np.exp(-pred_test))\ndf_sub.prediction = prob_test.loc[df_sub.customer_ID].values\n\ndf_sub.set_index('customer_ID').to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T14:06:25.453957Z","iopub.execute_input":"2022-05-27T14:06:25.454210Z","iopub.status.idle":"2022-05-27T14:06:38.133200Z","shell.execute_reply.started":"2022-05-27T14:06:25.454181Z","shell.execute_reply":"2022-05-27T14:06:38.131858Z"},"trusted":true},"execution_count":null,"outputs":[]}]}