{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What did I change:\n- Cloned from https://www.kaggle.com/code/lucasmorin/amex-woe-baseline\n\n- Replace the original Private dataset to my newly created dataset:\nhttps://www.kaggle.com/competitions/amex-default-prediction/discussion/327228\n\n- Notice that in the original WoE baseline notebook, the private dataset contains only 354 columns after prepare_df. which indicates there were some feature selection offline.","metadata":{}},{"cell_type":"markdown","source":"Simple weight of evidence baseline; WoE is a target encoding technique replacing values by an associated value that has nice additive properties.\nGive strong baseline, generally at the cost of feature interactions.\n\n**Don't Forget to upvote if you find this interesting or usefull**","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n\nimport gc\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\n# matplotlib setting\nmpl.rcParams['figure.dpi'] = 200\nmpl.rcParams['axes.spines.top'] = False\nmpl.rcParams['axes.spines.right'] = False\n\n# pandas setting\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 1000)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-26T10:00:43.937888Z","iopub.execute_input":"2022-05-26T10:00:43.938784Z","iopub.status.idle":"2022-05-26T10:00:45.046715Z","shell.execute_reply.started":"2022-05-26T10:00:43.938689Z","shell.execute_reply":"2022-05-26T10:00:45.045999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate_data","metadata":{}},{"cell_type":"code","source":"def prepare_df(df):\n    df_out = df.groupby('customer_ID').agg([np.mean,np.std])\n    df_out.columns = [c[0]+'_'+c[1] for c in df_out.columns]\n    df_out = df_out.fillna(np.nanmean(df_out))\n    return df_out","metadata":{"execution":{"iopub.status.busy":"2022-05-26T10:00:45.049540Z","iopub.execute_input":"2022-05-26T10:00:45.049886Z","iopub.status.idle":"2022-05-26T10:00:45.055603Z","shell.execute_reply.started":"2022-05-26T10:00:45.049827Z","shell.execute_reply":"2022-05-26T10:00:45.055010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_data = pd.read_pickle('../input/ae-credit-id-encoded-dataset-fp16/id_encoded_fp16_train_data.pkl')\n# Do some feature selection first\ntrain_data.drop(columns=[\"S_2\"], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T10:00:45.056606Z","iopub.execute_input":"2022-05-26T10:00:45.057069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_grp = prepare_df(train_data)\ndel train_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_pickle('../input/ae-credit-id-encoded-dataset-fp16/id_encoded_train_labels.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_data = pd.read_pickle('../input/ae-credit-id-encoded-dataset-fp16/id_encoded_fp16_test_data.pkl')\n# Do some feature selection first\ntest_data.drop(columns=[\"S_2\"], inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_grp = prepare_df(test_data)\ndel test_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# weight of evidence\n\nStandard Credit Scoring Technique. Target encoding technique that replace feature value by an additive value that helps build credit Scorecards. Personal sklearn implementation (doesn't handle edge case very well).","metadata":{}},{"cell_type":"code","source":"class WoE_Imputer(BaseEstimator, TransformerMixin):\n# Bins the features and impute Weight of Evidence associated with each bin\n# Weight of Evidence is calculated as the log ratio of positive outcome to negative ones in each bin\n# This imputation technique is adapted to the specific functionnal form of logistic regression\n# Allows to impute missing values\n# Also allows to calculate Information Values for feature selection\n    def __init__(self, feature_name, n_bin = 100, Categorical = False, verbosity = 1):  \n        self.feature_name = feature_name\n        self.n_bin = n_bin\n        self.bins = []\n        self.WoE_values = []\n        self.Categorical = Categorical \n        self.verbosity = verbosity\n        self.IV = 0\n\n    def fit(self, X, y = None):\n        if y is None:\n            raise ValueError('Woe Imputer is a supervised imputer. It needs a target')\n\n        if self.Categorical:\n            values_quantiles = X[self.feature_name].astype('category')\n            self.bins = values_quantiles.cat\n        else:\n            values_quantiles, self.bins = pd.qcut(X[self.feature_name], q=self.n_bin, duplicates = 'drop', retbins=True)   \n            self.bins[0] = -np.Inf\n            self.bins[-1] = np.Inf\n            values_quantiles = pd.cut(X[self.feature_name], bins = self.bins)\n\n        values_quantiles = values_quantiles.cat.add_categories('missing_value')\n        values_quantiles.fillna('missing_value', inplace = True) \n\n        df = pd.DataFrame({'group': values_quantiles, 'val': X[self.feature_name], 'target': y.values.flatten()})\n\n        sum_positive_by_quantile = df.groupby('group').sum().target\n        sum_negative_by_quantile = df.groupby('group').count().target - df.groupby('group').sum().target\n\n        data = np.log(sum_positive_by_quantile / sum_negative_by_quantile)\n        \n        #interpolate in case of na - there are other tricks\n        mask = np.isnan(data)\n        data[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), data[~mask])\n\n        self.WoE_values =  data\n\n        self.IV = ((sum_positive_by_quantile - sum_negative_by_quantile) * self.WoE_values / df.shape[0]).sum()\n\n        if self.verbosity>0:\n            print('Information Value ' + str(self.feature_name)+': ' + str(round(self.IV,5)))\n            \n        return self\n\n    def transform(self, X):\n        feature_to_transform = X[self.feature_name].copy()\n        transformed_feature = pd.cut(feature_to_transform, bins =  self.bins, labels = np.array(self.WoE_values[:-1]), ordered = False).astype('float32')\n        transformed_feature = transformed_feature.replace(np.nan, self.WoE_values[-1])\n        X[self.feature_name] = transformed_feature\n        return X\n\n    def __get_val__(self):  \n        return self.feature_name, self.n_bin, self.bins, self.WoE_values, self.IV","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm_notebook\n\nFeatures = train_data_grp.columns\nFeatures = [f for f in Features if not f.startswith('target')]\n\nIV_list = []\n\nfor f in tqdm_notebook(Features):\n    WoE_imp = WoE_Imputer(f, n_bin = 50, verbosity = 0)\n    WoE_imp.fit(train_data_grp, y = train_labels.target)\n    train_data_grp = WoE_imp.transform(train_data_grp)\n    test_data_grp = WoE_imp.transform(test_data_grp)\n    feature_name, n_bin, bins, WoE_values, IV = WoE_imp.__get_val__()\n    IV_list.append(IV)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_IV = pd.DataFrame({'Features':Features,'IV':IV_list}).sort_values('IV',ascending=False).reset_index(drop=True)\nplt.plot(sorted_IV.IV);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IV_threshold = 1.2\nlist_features = sorted_IV[sorted_IV.IV>IV_threshold].Features.to_list()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"pos / (pos + neg) = 1/(1+neg/pos) = (1/1+exp(-log(pos/neg)))","metadata":{}},{"cell_type":"code","source":"pred_train = train_data_grp[list_features].mean(axis=1)\nprob_train = 1/(1+np.exp(-pred_train))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric(train_labels.set_index('customer_ID'), prob_train.rename('prediction'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"df_sub = pd.read_pickle('../input/ae-credit-id-encoded-dataset-fp16/id_encoded_sample_submission.pkl')\n\npred_test = test_data_grp[list_features].mean(axis=1)\nprob_test = 1/(1+np.exp(-pred_test))\ndf_sub.prediction = prob_test.values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nloaded_encoder = LabelEncoder()\nloaded_encoder.classes_ = np.load(f\"../input/ae-credit-id-encoded-dataset-fp16/id_encodings.npy\", allow_pickle=True)\ndf_sub[\"customer_ID\"] = loaded_encoder.inverse_transform(df_sub[\"customer_ID\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub.set_index('customer_ID').to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}