{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"markdown","source":"# LightGBM Quickstart for the *American Express - Default Prediction* competition\n\nThis notebook shows how to apply LightGBM to the competition data.\n\nIt is based on the [EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense).","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.calibration import CalibrationDisplay\nfrom lightgbm import LGBMClassifier, log_evaluation\n\n# plt.rcParams['axes.facecolor'] = '#0057b8' # blue\n# plt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n#                                          plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\n# plt.rcParams['text.color'] = 'w'","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-05-28T22:35:37.877938Z","iopub.execute_input":"2022-05-28T22:35:37.878501Z","iopub.status.idle":"2022-05-28T22:35:39.748769Z","shell.execute_reply.started":"2022-05-28T22:35:37.878403Z","shell.execute_reply":"2022-05-28T22:35:39.748265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/code/inversion/amex-competition-metric-python\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        \"\"\"Almost equal to 2 * auc - 1\"\"\"\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n#     print(f\"{g:.5f} {d:.5f}\")\n\n    return 0.5 * (g + d)\n\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(pd.DataFrame({'target': y_true}), pd.Series(y_pred, name='prediction')),\n            True)\n\n#From : https://www.kaggle.com/competitions/amex-default-prediction/discussion/327534\ndef amex_metric_mod_lgbm(y_pred: np.ndarray, data: lgb.Dataset):\n\n    y_true = data.get_label()\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 'AMEX', 0.5 * (gini[1]/gini[0]+ top_four), True","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T22:35:39.749935Z","iopub.execute_input":"2022-05-28T22:35:39.750264Z","iopub.status.idle":"2022-05-28T22:35:39.766917Z","shell.execute_reply.started":"2022-05-28T22:35:39.750241Z","shell.execute_reply":"2022-05-28T22:35:39.766194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading and preprocessing the data\n\nWe read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). Then we reduce the amount of data by keeping only the most recent statement for every customer, as suggested by @inversion [here](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327094).","metadata":{}},{"cell_type":"code","source":"%%time\ntrain = pd.read_feather('../input/amexfeather/train_data.ftr')\ntest = pd.read_feather('../input/amexfeather/test_data.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:35:39.767871Z","iopub.execute_input":"2022-05-28T22:35:39.768093Z","iopub.status.idle":"2022-05-28T22:36:28.266174Z","shell.execute_reply.started":"2022-05-28T22:35:39.768072Z","shell.execute_reply":"2022-05-28T22:36:28.265309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train =  (train\n            .groupby('customer_ID')\n            .tail(1)\n            .set_index('customer_ID', drop=True)\n            .sort_index()\n            .drop(['S_2'], axis='columns'))\n\ntest =  (test\n            .groupby('customer_ID')\n            .tail(1)\n            .set_index('customer_ID', drop=True)\n            .sort_index()\n            .drop(['S_2'], axis='columns'))","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:36:28.267922Z","iopub.execute_input":"2022-05-28T22:36:28.268368Z","iopub.status.idle":"2022-05-28T22:36:34.441541Z","shell.execute_reply.started":"2022-05-28T22:36:28.268335Z","shell.execute_reply":"2022-05-28T22:36:34.441027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **EDA**","metadata":{}},{"cell_type":"markdown","source":"# The categorical features\n\nAccording to the [data description](https://www.kaggle.com/competitions/amex-default-prediction/data), there are eleven categorical features. We plot histograms for target=0 and target=1:\n\n - customer_ID = Unique Customer ID\n - D_* = Delinquency variables\n - S_* = Spend variables\n - P_* = Payment variables\n - B_* = Balance variables\n - R_* = Risk variables\n","metadata":{}},{"cell_type":"code","source":"cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nplt.figure(figsize=(16, 16))\nfor i, f in enumerate(cat_features):\n    plt.subplot(4, 3, i+1)\n    temp = pd.DataFrame(train[f][train.target == 0].value_counts(dropna=False, normalize=True).sort_index().rename('count'))\n    temp.index.name = 'value'\n    temp.reset_index(inplace=True)\n    plt.bar(temp.index, temp['count'], alpha=0.5, label='target=0')\n    temp = pd.DataFrame(train[f][train.target == 1].value_counts(dropna=False, normalize=True).sort_index().rename('count'))\n    temp.index.name = 'value'\n    temp.reset_index(inplace=True)\n    plt.bar(temp.index, temp['count'], alpha=0.5, label='target=1')\n    plt.xlabel(f)\n    plt.ylabel('frequency')\n    plt.legend()\n    plt.xticks(temp.index, temp.value)\nplt.show()\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T22:36:34.442453Z","iopub.execute_input":"2022-05-28T22:36:34.442780Z","iopub.status.idle":"2022-05-28T22:36:36.347370Z","shell.execute_reply.started":"2022-05-28T22:36:34.442757Z","shell.execute_reply":"2022-05-28T22:36:36.346773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Insight:**\n- Every feature has at most eight categories (including a nan category). One-hot encodings are feasible.\n- The distributions for target=0 and target=1 differ. This means that every feature gives some information about the target.\n- Some features are 0,1, or nans \n- D_66 is mainly 1.0 or Nan, it can be treated as a binary\n","metadata":{}},{"cell_type":"markdown","source":"# The binary features\n\nTwo features are binary:\n- B_31 is always 0 or 1.\n- D_87 and D_66 are always 1 or missing.\n","metadata":{}},{"cell_type":"code","source":"bin_features = ['B_31', 'D_87', 'D_66']\nplt.figure(figsize=(16, 4))\nfor i, f in enumerate(bin_features):\n    plt.subplot(1, 3, i+1)\n    temp = pd.DataFrame(train[f][train.target == 0].value_counts(dropna=False, normalize=True).sort_index().rename('count'))\n    temp.index.name = 'value'\n    temp.reset_index(inplace=True)\n    plt.bar(temp.index, temp['count'], alpha=0.5, label='target=0')\n    temp = pd.DataFrame(train[f][train.target == 1].value_counts(dropna=False, normalize=True).sort_index().rename('count'))\n    temp.index.name = 'value'\n    temp.reset_index(inplace=True)\n    plt.bar(temp.index, temp['count'], alpha=0.5, label='target=1')\n    plt.xlabel(f)\n    plt.ylabel('frequency')\n    plt.legend()\n    plt.xticks(temp.index, temp.value)\nplt.show()\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T22:36:36.348454Z","iopub.execute_input":"2022-05-28T22:36:36.348855Z","iopub.status.idle":"2022-05-28T22:36:36.709293Z","shell.execute_reply.started":"2022-05-28T22:36:36.348806Z","shell.execute_reply":"2022-05-28T22:36:36.708385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The continuous features\n\nIf we plot histograms of the continuous features, we see that they have all kinds of distributions:","metadata":{}},{"cell_type":"code","source":"cont_features = sorted([f for f in train.columns if f not in cat_features + bin_features + ['customer_ID', 'target']])\nprint(cont_features, end = '')\nncols = 5\nfor i, f in enumerate(cont_features):\n    #show 4 rows\n    if i >=20:\n        break\n    if i % ncols == 0: \n        if i > 0: plt.show()\n        plt.subplots(1, ncols, figsize=(16, 3))\n    plt.subplot(1, ncols, i % ncols + 1)\n    \n    sns.histplot(data=train, x=f, hue=\"target\", bins = 100, kde=True)\n    \n    plt.xlabel(f)\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-28T22:36:36.710495Z","iopub.execute_input":"2022-05-28T22:36:36.710947Z","iopub.status.idle":"2022-05-28T22:37:36.377341Z","shell.execute_reply.started":"2022-05-28T22:36:36.710916Z","shell.execute_reply":"2022-05-28T22:37:36.376213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Insight:** Histograms with white space at the left or right end indicate that the data contains outliers. We will have to deal with these outliers.\n\nWe also need to look for correlations in order to impute the missing values later","metadata":{}},{"cell_type":"markdown","source":"# Impute Continuous \n\nImpute with median","metadata":{}},{"cell_type":"code","source":"%%time\nfrom sklearn.impute import SimpleImputer\n# from sklearn.experimental import enable_iterative_imputer\n# from sklearn.impute import IterativeImputer\n# from sklearn.neighbors import KNeighborsRegressor\ncontinuous_imputer = SimpleImputer(strategy = 'median')\n\n\nprint('Fit')\ncontinuous_imputer.fit(train[cont_features])\nprint('Pred')\ntrain[cont_features] = continuous_imputer.transform(train[cont_features])\ntest[cont_features] = continuous_imputer.transform(test[cont_features])\nprint('Done')\n\ndel continuous_imputer","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:37:36.378539Z","iopub.execute_input":"2022-05-28T22:37:36.378834Z","iopub.status.idle":"2022-05-28T22:37:53.810191Z","shell.execute_reply.started":"2022-05-28T22:37:36.378782Z","shell.execute_reply":"2022-05-28T22:37:53.809205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Countinuous Correlations\n\nAs there are too many columns to calculate, we must calculate correlations one row at a time. Otherwise it would take too much memory\n\nThis also takes very long, so in the interest of time I copy-pasted the calculations in the code cell below the next","metadata":{}},{"cell_type":"code","source":"%%time\ncorr_dict = {x : [] for x in cont_features}\n# corr_df = pd.DataFrame(index = cont_features, columns = cont_features)\nfor col in cont_features:\n    corr_dict[col] = train[cont_features].corrwith(train[col]).to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:37:53.811367Z","iopub.execute_input":"2022-05-28T22:37:53.811596Z","iopub.status.idle":"2022-05-28T22:42:25.585890Z","shell.execute_reply.started":"2022-05-28T22:37:53.811574Z","shell.execute_reply":"2022-05-28T22:42:25.584854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from numpy import nan\n\nnum_corr = pd.DataFrame.from_dict(corr_dict)\nnum_corr.index = cont_features\nnum_corr","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:42:25.588380Z","iopub.execute_input":"2022-05-28T22:42:25.588675Z","iopub.status.idle":"2022-05-28T22:42:25.620348Z","shell.execute_reply.started":"2022-05-28T22:42:25.588652Z","shell.execute_reply":"2022-05-28T22:42:25.619862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del corr_dict\ndel num_corr","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:42:25.621136Z","iopub.execute_input":"2022-05-28T22:42:25.621439Z","iopub.status.idle":"2022-05-28T22:42:25.623666Z","shell.execute_reply.started":"2022-05-28T22:42:25.621418Z","shell.execute_reply":"2022-05-28T22:42:25.623278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Treat Contiuous Outliers","metadata":{}},{"cell_type":"code","source":"%%time\nfor col in cont_features:\n#     mean, std = train[col].astype(np.float64).agg(['mean', 'std'])\n#     low  = mean - 3*std\n#     high = mean + 3*std\n#     train.loc[train[col] < low,:] = low\n#     train.loc[train[col] > high,:] = high\n#     test.loc[test[col] < low,:] = low\n#     test.loc[test[col] > high,:] = high\n    tenth_percentile = np.percentile(train[col], 10)\n    ninetieth_percentile = np.percentile(train[col], 95)\n    train[col] = np.where(train[col]<tenth_percentile, tenth_percentile, train[col])\n    train[col] = np.where(train[col]>ninetieth_percentile, ninetieth_percentile, train[col])\n    \n    tenth_percentile = np.percentile(test[col], 3)\n    ninetieth_percentile = np.percentile(test[col], 97)\n    test[col] = np.where(test[col]<tenth_percentile, tenth_percentile, test[col])\n    test[col] = np.where(test[col]>ninetieth_percentile, ninetieth_percentile, test[col])","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:42:25.624691Z","iopub.execute_input":"2022-05-28T22:42:25.625013Z","iopub.status.idle":"2022-05-28T22:42:36.428939Z","shell.execute_reply.started":"2022-05-28T22:42:25.624990Z","shell.execute_reply":"2022-05-28T22:42:36.428293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ncols = 5\nfor i, f in enumerate(cont_features):\n    #show 2 rows\n    if i >=10:\n        break\n    if i % ncols == 0: \n        if i > 0: plt.show()\n        plt.subplots(1, ncols, figsize=(16, 3))\n    plt.subplot(1, ncols, i % ncols + 1)\n    sns.histplot(data=train, x=f, hue=\"target\", bins = 100, kde=True)\n    plt.xlabel(f)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:42:36.429870Z","iopub.execute_input":"2022-05-28T22:42:36.430098Z","iopub.status.idle":"2022-05-28T22:43:02.505479Z","shell.execute_reply.started":"2022-05-28T22:42:36.430076Z","shell.execute_reply":"2022-05-28T22:43:02.504758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Binary Preprocessing\n\nNeed to turn D_87 and D_66 into 0s and 1s.","metadata":{}},{"cell_type":"code","source":"train[['D_87','D_66']] = train[['D_87','D_66']].astype('float').fillna(0)\ntest[['D_87','D_66']] = test[['D_87','D_66']].astype('float').fillna(0)\n\ntrain[['D_87','D_66']].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:43:02.506412Z","iopub.execute_input":"2022-05-28T22:43:02.506642Z","iopub.status.idle":"2022-05-28T22:43:02.681840Z","shell.execute_reply.started":"2022-05-28T22:43:02.506620Z","shell.execute_reply":"2022-05-28T22:43:02.680945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categorical Correlations\n\nUse Cramers V for correlations calculations\n\nInclude binary variables too","metadata":{}},{"cell_type":"code","source":"# Label Encode Categorical Variables\nfrom sklearn.preprocessing import LabelEncoder\n\n\ntrain_cat = train[['B_30', 'B_31', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68', 'D_87']].astype('object')\n\nlabel = LabelEncoder()\n\nfor i in train_cat.columns:\n    train_cat[i] = label.fit_transform(train_cat[i])\n    \ntrain_cat.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:43:02.683054Z","iopub.execute_input":"2022-05-28T22:43:02.683413Z","iopub.status.idle":"2022-05-28T22:43:04.119597Z","shell.execute_reply.started":"2022-05-28T22:43:02.683380Z","shell.execute_reply":"2022-05-28T22:43:04.118868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats.contingency import association       \n    \ndef Cramers_V(var1, var2) :\n  crosstab = np.array(pd.crosstab(index=var1, columns=var2)) # Cross Tab\n  return (association(crosstab, method='cramer'))            # Return Cramer's V\n\n# Create the dataFrame matrix with the returned Cramer's V\nrows = []\n\nfor var1 in train_cat:\n    col = []\n\n    for var2 in train_cat:\n        V = Cramers_V(train_cat[var1], train_cat[var2]) # Return Cramer's V\n        col.append(V)                                             # Store values to subsequent columns  \n    rows.append(col)                                              # Store values to subsequent rows\n  \nCramersV_results = np.array(rows)\nCramersV_df = pd.DataFrame(CramersV_results, columns = train_cat.columns, index = train_cat.columns)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:43:04.120804Z","iopub.execute_input":"2022-05-28T22:43:04.121503Z","iopub.status.idle":"2022-05-28T22:43:18.931120Z","shell.execute_reply.started":"2022-05-28T22:43:04.121471Z","shell.execute_reply":"2022-05-28T22:43:18.930551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20,7))\nsns.heatmap(CramersV_df, cmap='cool', annot=True, center=0)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:43:18.933780Z","iopub.execute_input":"2022-05-28T22:43:18.934028Z","iopub.status.idle":"2022-05-28T22:43:19.571261Z","shell.execute_reply.started":"2022-05-28T22:43:18.934007Z","shell.execute_reply":"2022-05-28T22:43:19.570520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Insight**: most features have significant correlations to other categorical features.\n\nUse categorical features to impute each other's NaN values","metadata":{}},{"cell_type":"markdown","source":"# Impute Categorical\n\nUse KNNImputer to impute columns using their label encodings\n","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.impute import KNNImputer\ncat_features = ['B_30', 'B_31', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68', 'D_87']\n\nencoder = LabelEncoder()\ncat_imputer = KNNImputer(n_neighbors=1)\n\ntrain_cat = train[cat_features]\ntest_cat = test[cat_features]\n\ntrain_cat = train_cat.apply(lambda series: pd.Series(\n    encoder.fit_transform(series[series.notnull()]),\n    index=series[series.notnull()].index\n))\ntest_cat = test_cat.apply(lambda series: pd.Series(\n    encoder.fit_transform(series[series.notnull()]),\n    index=series[series.notnull()].index\n))\ntrain_cat =  pd.DataFrame(train_cat ,columns = cat_features, index = train.index).astype(float)\ntest_cat =  pd.DataFrame(test_cat ,columns = cat_features, index = test.index).astype(float)\ntrain_cat.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:43:19.572070Z","iopub.execute_input":"2022-05-28T22:43:19.572334Z","iopub.status.idle":"2022-05-28T22:43:28.644986Z","shell.execute_reply.started":"2022-05-28T22:43:19.572305Z","shell.execute_reply":"2022-05-28T22:43:28.643930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_imputer.fit(train_cat.head(50000))\ntrain_cat = cat_imputer.transform(train_cat)\ntest_cat = cat_imputer.transform(test_cat)\n\ntrain_cat =  pd.DataFrame(train_cat ,columns = cat_features).astype(int)\ntest_cat =  pd.DataFrame(test_cat ,columns = cat_features).astype(int)\n\ntrain_cat.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:43:28.646023Z","iopub.execute_input":"2022-05-28T22:43:28.646490Z","iopub.status.idle":"2022-05-28T22:45:16.134080Z","shell.execute_reply.started":"2022-05-28T22:43:28.646460Z","shell.execute_reply":"2022-05-28T22:45:16.133249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_features:\n    encoder.fit(train[col][train[col].notnull()])\n    train_cat[col] = encoder.inverse_transform(train_cat[col])\n    encoder.fit(test[col][test[col].notnull()])\n    test_cat[col] = encoder.inverse_transform(test_cat[col])\n\ntrain_cat","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:16.135291Z","iopub.execute_input":"2022-05-28T22:45:16.135567Z","iopub.status.idle":"2022-05-28T22:45:17.563481Z","shell.execute_reply.started":"2022-05-28T22:45:16.135541Z","shell.execute_reply":"2022-05-28T22:45:17.562773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cat.index = train.index\ntest_cat.index = test.index\ntrain[cat_features] = train_cat\ntest[cat_features] = test_cat\n# sort array columns for organization\ntrain = train.sort_index(axis=1)\ntest = test.sort_index(axis=1)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:17.564361Z","iopub.execute_input":"2022-05-28T22:45:17.565092Z","iopub.status.idle":"2022-05-28T22:45:18.156315Z","shell.execute_reply.started":"2022-05-28T22:45:17.565069Z","shell.execute_reply":"2022-05-28T22:45:18.155496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_cat\ndel test_cat\ndel cat_imputer\ndel encoder","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:18.157772Z","iopub.execute_input":"2022-05-28T22:45:18.158108Z","iopub.status.idle":"2022-05-28T22:45:18.167295Z","shell.execute_reply.started":"2022-05-28T22:45:18.158079Z","shell.execute_reply":"2022-05-28T22:45:18.166361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Preprocessing\n\nFinally prep data for modeling","metadata":{}},{"cell_type":"code","source":"#one-hot encode the non-binary categorical features\ncat_cols = [col for col in cat_features if train[col].nunique()>2]\n\nfrom sklearn.preprocessing import OneHotEncoder\nohenc = OneHotEncoder(sparse=False)\nOH_cols_train = pd.DataFrame(ohenc.fit_transform(train[cat_cols]))\nOH_cols_test = pd.DataFrame(ohenc.transform(test[cat_cols]))\n\nOH_cols_train.columns = ohenc.get_feature_names_out()\nOH_cols_test.columns = ohenc.get_feature_names_out()\n\nOH_cols_train.index = train.index\nOH_cols_test.index = test.index\n\ntrain = train.drop(cat_cols, axis=1)\ntest = test.drop(cat_cols, axis=1)\n\ntrain = pd.concat([train, OH_cols_train], axis=1)\ntest = pd.concat([test, OH_cols_test], axis=1)\n\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:18.168447Z","iopub.execute_input":"2022-05-28T22:45:18.168756Z","iopub.status.idle":"2022-05-28T22:45:21.438503Z","shell.execute_reply.started":"2022-05-28T22:45:18.168688Z","shell.execute_reply":"2022-05-28T22:45:21.437763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del OH_cols_train\ndel OH_cols_test","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:21.439456Z","iopub.execute_input":"2022-05-28T22:45:21.439659Z","iopub.status.idle":"2022-05-28T22:45:21.459602Z","shell.execute_reply.started":"2022-05-28T22:45:21.439639Z","shell.execute_reply":"2022-05-28T22:45:21.459120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Helpfullness of Each Feature for Modelling\n\nI will be using LGBMClassifier for modeling.\n\nCheck the Permutation Importance of each feature","metadata":{}},{"cell_type":"code","source":"# import eli5\n# from lightgbm import LGBMRegressor\n# from eli5.sklearn import PermutationImportance\n# from sklearn.model_selection import train_test_split\n\n# train_X, val_X, train_y, val_y = train_test_split(train.drop('target',axis = 1).astype('float64'),train['target'].astype('float64'), random_state=1,test_size =0.1)\n\n# model = LGBMRegressor(n_estimators=170,\n#                       objective = 'binary',\n#                       min_child_samples=2400,\n#                       num_leaves=127,\n#                       max_bins=511, random_state=1)\n\n\n# model.fit(train_X,train_y)\n\n# perm = PermutationImportance(model, random_state=1).fit(val_X, val_y)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:21.460548Z","iopub.execute_input":"2022-05-28T22:45:21.460763Z","iopub.status.idle":"2022-05-28T22:45:21.471082Z","shell.execute_reply.started":"2022-05-28T22:45:21.460743Z","shell.execute_reply":"2022-05-28T22:45:21.470587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eli5.show_weights(perm, feature_names = val_X.columns.tolist(),top=188)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:21.471682Z","iopub.execute_input":"2022-05-28T22:45:21.472385Z","iopub.status.idle":"2022-05-28T22:45:21.482580Z","shell.execute_reply.started":"2022-05-28T22:45:21.472363Z","shell.execute_reply":"2022-05-28T22:45:21.482139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_fold = 2\nn_seed = 2\nn_estimators = 100\n\nkf = StratifiedKFold(n_splits=n_fold)\n\nimportances = []\nmodels = {}\ndf_scores = []\n\nSAMPLE = False\n\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train.drop('target',axis = 1).astype('float64'),train[['target']])):\n    \n    X_tr = train.drop('target',axis = 1).iloc[idx_tr]\n    X_va = train.drop('target',axis = 1).iloc[idx_va]\n    y_tr = train[['target']].iloc[idx_tr]\n    y_va = train[['target']].iloc[idx_va]\n    \n    lgb_train_data = lgb.Dataset(X_tr, label=y_tr)\n    lgb_val_data = lgb.Dataset(X_va, label=y_va)\n    \n    for seed in range(n_seed):\n        print('Fold: '+str(fold)+ ' - seed: '+str(seed))\n        key = str(fold)+'-'+str(seed)\n        \n        parameters = {\n            'objective': 'binary',\n            'boosting': 'gbdt',\n            'learning_rate': 0.05,\n            'min_child_samples': 1000,\n            'reg_lambda':10,\n            #'verbose': 25,\n            'seed':seed,\n            'n_estimators':n_estimators\n        }\n\n        clf = lgb.train(parameters,\n                               lgb_train_data,\n                               valid_sets = [lgb_train_data,lgb_val_data],\n                               verbose_eval = 100,\n                               feval=amex_metric_mod_lgbm,\n                               early_stopping_rounds=50)\n\n        score = amex_metric(y_va.reset_index(drop=True), pd.Series(clf.predict(X_va)).rename('prediction'))\n        models[key] = clf\n        df_scores.append((fold, seed, score))\n        print(f'Fold: {fold} - seed: {seed} - score {score:.2%}')\n        importances.append(clf.feature_importance(importance_type='gain'))","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:21.483309Z","iopub.execute_input":"2022-05-28T22:45:21.483618Z","iopub.status.idle":"2022-05-28T22:48:02.411007Z","shell.execute_reply.started":"2022-05-28T22:45:21.483593Z","shell.execute_reply":"2022-05-28T22:48:02.410149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_results = pd.DataFrame(df_scores,columns=['fold','seed','score']).pivot(index='fold',columns='seed',values='score')\n\ndf_results.loc['seed_mean']= df_results.mean(numeric_only=True, axis=0)\ndf_results.loc[:,'fold_mean'] = df_results.mean(numeric_only=True, axis=1)\ndf_results","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:48:02.414212Z","iopub.execute_input":"2022-05-28T22:48:02.414504Z","iopub.status.idle":"2022-05-28T22:48:02.433531Z","shell.execute_reply.started":"2022-05-28T22:48:02.414482Z","shell.execute_reply":"2022-05-28T22:48:02.432727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_importance(importances, features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()\n    \nplot_importance(np.array(importances),train.drop('target',axis=1).columns, PLOT_TOP_N = 20, figsize=(10, 20))","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:48:02.434906Z","iopub.execute_input":"2022-05-28T22:48:02.435480Z","iopub.status.idle":"2022-05-28T22:48:03.291648Z","shell.execute_reply.started":"2022-05-28T22:48:02.435447Z","shell.execute_reply":"2022-05-28T22:48:03.290654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Recalibrate\n\nKeep most important features","metadata":{}},{"cell_type":"code","source":"top_ratio = 3/4\nimportance_df = pd.DataFrame(data=importances,columns =  train.drop('target',axis=1).columns).median(axis=0).sort_values(ascending=False)\nkeep_features = importance_df.index[:int(len(importance_df.index)*top_ratio)].to_list()\nFeatures = keep_features","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:48:03.292635Z","iopub.execute_input":"2022-05-28T22:48:03.292866Z","iopub.status.idle":"2022-05-28T22:48:03.362351Z","shell.execute_reply.started":"2022-05-28T22:48:03.292844Z","shell.execute_reply":"2022-05-28T22:48:03.361572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross-validation\n\nWe cross-validate with a five-fold StratifiedKFold.\n\nNotice that lightgbm logs the validation score with the competition's scoring function every ten iterations.","metadata":{}},{"cell_type":"code","source":"#%%time\n# Cross-validation of the classifier\n\nINFERENCE = True\n\ndef my_booster(random_state=1, n_estimators=300):\n    return LGBMClassifier(n_estimators=n_estimators,\n                          min_child_samples=2400,\n                          num_leaves=127,\n                          max_bins=511, random_state=random_state)\n      \nprint(f\"{len(Features)} features\")\nscore_list = []\ny_pred_list = []\nkf = StratifiedKFold(n_splits=5)\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train, train.target)):\n    start_time = datetime.datetime.now()\n    X_tr = train.iloc[idx_tr][Features]\n    X_va = train.iloc[idx_va][Features]\n    y_tr = train.iloc[idx_tr].target\n    y_va = train.iloc[idx_va].target\n    \n    model = my_booster()\n    with warnings.catch_warnings():\n        warnings.filterwarnings('ignore', category=UserWarning)\n        model.fit(X_tr, y_tr,\n                  eval_set = [(X_tr, y_tr),(X_va, y_va)], \n                  eval_metric=[lgb_amex_metric],\n                  callbacks=[log_evaluation(10)])\n    y_va_pred = model.predict_proba(X_va)[:,1]\n    score = amex_metric(pd.DataFrame({'target': y_va.values}), pd.Series(y_va_pred, name='prediction'))\n    n_trees = model.best_iteration_\n    if n_trees is None: n_trees = model.n_estimators\n    print(f\"Fold {fold} | {str(datetime.datetime.now() - start_time)[-12:-7]} |\"\n          f\" {n_trees:5} trees |\"\n          f\"                Score = {score:.5f}\")\n    score_list.append(score)\n    evals_result = model.evals_result_\n    \n    if INFERENCE:\n        y_pred_list.append(model.predict_proba(test[Features])[:,1])\n        \n    # break # we only want the first fold\n    \nprint(f\"OOF Score:                       {np.mean(score_list):.5f}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:50:54.624593Z","iopub.execute_input":"2022-05-28T22:50:54.625474Z","iopub.status.idle":"2022-05-28T23:22:26.905580Z","shell.execute_reply.started":"2022-05-28T22:50:54.625430Z","shell.execute_reply":"2022-05-28T23:22:26.904867Z"},"jupyter":{"source_hidden":true,"outputs_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calibration diagram\n\nThe calibration diagram shows how the model predicts the default probability of customers:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nCalibrationDisplay.from_predictions(y_va, y_va_pred, n_bins=50, strategy='quantile', ax=plt.gca())\nplt.title('Probability calibration')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T23:27:51.754426Z","iopub.execute_input":"2022-05-28T23:27:51.754789Z","iopub.status.idle":"2022-05-28T23:27:51.931107Z","shell.execute_reply.started":"2022-05-28T23:27:51.754763Z","shell.execute_reply":"2022-05-28T23:27:51.929950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\n\nMake an ensemble","metadata":{}},{"cell_type":"code","source":"if INFERENCE:\n    sub = pd.DataFrame({'customer_ID': test.index,\n                        'prediction': np.mean(y_pred_list, axis=0)})\n    sub.to_csv('submission.csv', index=False)\n    sub","metadata":{"execution":{"iopub.status.busy":"2022-05-28T23:27:59.834899Z","iopub.execute_input":"2022-05-28T23:27:59.835234Z","iopub.status.idle":"2022-05-28T23:28:02.537438Z","shell.execute_reply.started":"2022-05-28T23:27:59.835211Z","shell.execute_reply":"2022-05-28T23:28:02.536866Z"},"trusted":true},"execution_count":null,"outputs":[]}]}