{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AMEX - lgbm + Feature Engineering\n","metadata":{"papermill":{"duration":0.00979,"end_time":"2022-01-02T19:15:51.734235","exception":false,"start_time":"2022-01-02T19:15:51.724445","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport pickle\n\nimport time\nfrom datetime import datetime\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport lightgbm as lgb\n\nimport warnings\n\nfrom sklearn.model_selection import StratifiedKFold\n\nbase_seed = 0\n\nDEBUG = True\n\nif ~DEBUG:\n    warnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":2.402335,"end_time":"2022-01-02T19:15:54.14708","exception":false,"start_time":"2022-01-02T19:15:51.744745","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-27T22:41:14.60974Z","iopub.execute_input":"2022-05-27T22:41:14.610235Z","iopub.status.idle":"2022-05-27T22:41:16.774971Z","shell.execute_reply.started":"2022-05-27T22:41:14.61013Z","shell.execute_reply":"2022-05-27T22:41:16.773918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 1000)\n\nDEBUG = False","metadata":{"execution":{"iopub.status.busy":"2022-05-27T22:41:16.776753Z","iopub.execute_input":"2022-05-27T22:41:16.777241Z","iopub.status.idle":"2022-05-27T22:41:16.783466Z","shell.execute_reply.started":"2022-05-27T22:41:16.7772Z","shell.execute_reply":"2022-05-27T22:41:16.782026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_pickle('../input/amex-training-pickle-dataset/train_data.pkl').iloc[:100000]\ntrain_data1 = pd.read_pickle('../input/amex-feature-engineering/train_data_agg.pkl').iloc[:100000]\n\nif DEBUG:\n    # around 2% of data\n    train_data1 = train_data1.iloc[:100000]\ntrain_labels = pd.read_pickle('../input/amex-training-pickle-dataset/train_labels.pkl').iloc[:100000]\ntrain_labels1 = pd.read_csv('../input/amex-default-prediction/train_labels.csv').iloc[:100000]\nFeatures = train_data1.columns","metadata":{"execution":{"iopub.status.busy":"2022-05-27T22:41:16.784912Z","iopub.execute_input":"2022-05-27T22:41:16.78547Z","iopub.status.idle":"2022-05-27T22:41:49.323435Z","shell.execute_reply.started":"2022-05-27T22:41:16.785428Z","shell.execute_reply":"2022-05-27T22:41:49.321699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Feature engineering\ntest_read = pd.read_csv('../input/amex-default-prediction/train_data.csv',nrows=10)\n\ntest_read.S_2 = (pd.to_datetime(test_read.S_2).max() - pd.to_datetime(test_read.S_2)).dt.days\ndate = ['customer_ID']\ncat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nbin_features = ['B_31', 'D_87']\n\nagg_dict_num = {}\nagg_dict_cat = {}\n\nagg_dict_cat['customer_ID'] = ['count']\n\nmean_diff = lambda x: np.nanmean(np.diff(x.values))\nmean_diff.__name__ = 'mean_diff'\n\nfor c in test_read.columns:\n    if c not in date:\n        if c not in cat_features+bin_features:\n            agg_dict_num[c] = ['mean','std','min','max','last'] #'median','skew', mean_diff] # too slow with pandas\n        else:\n            agg_dict_cat[c] = ['nunique'] # need 'last' too but with categorical encoding\n            \ndef prepare_df_num(df):\n    df.loc[:,'S_2'] = (pd.to_datetime(df.S_2).max() - pd.to_datetime(df.S_2)).dt.days\n    df_agg = df.groupby('customer_ID').agg(agg_dict_num)\n    df_agg.columns = [str(c[0])+'_'+str(c[1]) for c in df_agg.columns]\n    df_agg = df_agg.fillna(np.nanmean(df_agg))\n    return df_agg\n\ndef prepare_df_cat(df):\n    df.loc[:,cat_features+bin_features] = df.loc[:,cat_features+bin_features].astype(str)\n    df_agg = df.groupby('customer_ID').agg(agg_dict_cat)\n    df_agg.columns = [str(c[0])+'_'+str(c[1]) for c in df_agg.columns]\n    df_list = []\n    for c in cat_features+bin_features:\n        df_cat = df.groupby(['customer_ID',c])[c].count()\n        df_cat = df_cat.unstack()\n        df_cat.columns = [df_cat.columns.name + '_' + c for c in df_cat.columns]\n        df_cat = df_cat.fillna(0)\n        df_list.append(df_cat)\n    df_out = pd.concat([df_agg]+df_list, axis=1)\n    df_out = df_out.fillna(np.nanmean(df_out))\n    return df_out   \n\n# https://stackoverflow.com/questions/2130016/splitting-a-list-into-n-parts-of-approximately-equal-length\ndef split(a, n):\n    k, m = divmod(len(a), n)\n    return (a[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n))\n\nsplit_ids = split(train_data.customer_ID.unique(),10)\n\ndf_list_train = []\n\nfor (i,ids) in enumerate(split_ids):\n    print(i)\n    train_data_ids = train_data[train_data.customer_ID.isin(ids)]\n    train_data_num = prepare_df_num(train_data_ids).astype('float16')\n    train_data_cat = prepare_df_cat(train_data_ids).astype('float16')\n    df_list_train.append(pd.concat([train_data_num,train_data_cat],axis=1))\n    gc.collect()\n    \npd.concat(df_list_train,axis=0).astype('float16').to_pickle('train_data_agg.pkl')\n\ndel train_data, train_data_num, train_data_cat, train_data_ids, df_list_train\ngc.collect()\n\ntest_data = pd.read_csv('../input/amex-default-prediction/test_data.csv').iloc[:100000]\n\nif DEBUG:\n    test_data = test_data.iloc[:100000]\n    \nsplit_ids = split(test_data.customer_ID.unique(),10)\n\ndf_list_test = []\n\nfor (i,ids) in enumerate(split_ids):\n    print(i)\n    test_data_ids = test_data[test_data.customer_ID.isin(ids)]\n    test_data_num = prepare_df_num(test_data_ids).astype('float16')\n    test_data_cat = prepare_df_cat(test_data_ids).astype('float16')\n    df_list_test.append(pd.concat([test_data_num,test_data_cat],axis=1))\n    gc.collect()\n    \npd.concat(df_list_test,axis=0).astype('float16').to_pickle('test_data_agg.pkl')\n\ndel test_data, test_data_num, test_data_cat, test_data_ids, df_list_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T22:52:07.157289Z","iopub.execute_input":"2022-05-27T22:52:07.159189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n\n# from ambrosm notebook\ndef lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(pd.DataFrame({'target': y_true}), pd.Series(y_pred, name='prediction')),\n            True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_fold = 2 if DEBUG else 5\nn_seed = 2 if DEBUG else 5\nn_estimators = 200 if DEBUG else 1000\n\nkf = StratifiedKFold(n_splits=n_fold)\n\nimportances = []\nmodels = {}\ndf_scores = []\n\nSAMPLE = False\n\nfor fold, (idx_tr, idx_va) in enumerate(kf.split(train_data, train_labels)):\n    X_tr = train_data1.iloc[idx_tr]\n    X_va = train_data1.iloc[idx_va]\n    y_tr = train_labels1.iloc[idx_tr]\n    y_va = train_labels1.iloc[idx_va]\n    for seed in range(n_seed):\n        print('Fold: '+str(fold)+ ' - seed: '+str(seed))\n        clf = lgb.LGBMClassifier(min_child_samples=1000,\n            learning_rate=0.05, \n            n_estimators=n_estimators,\n            reg_lambda = 10,\n            class_weight = {1:0.01,0:1},\n            random_state=seed).fit(X_tr, y_tr, eval_set=[(X_tr,y_tr),(X_va,y_va)], verbose=25, early_stopping_rounds=50, eval_metric=[lgb_amex_metric])\n        \n        key = str(fold)+'-'+str(seed)\n        score = amex_metric(y_va.reset_index(drop=True), pd.Series(clf.predict_proba(X_va)[:,1]).rename('prediction'))\n        models[key] = clf\n        df_scores.append((fold, seed, score))\n        print(f'Fold: {fold} - seed: {seed} - score {score:.2%}')\n        importances.append(clf.booster_.feature_importance(importance_type='gain'))","metadata":{"papermill":{"duration":1432.413344,"end_time":"2022-01-02T19:39:46.616263","exception":false,"start_time":"2022-01-02T19:15:54.202919","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_results = pd.DataFrame(df_scores,columns=['fold','seed','score']).pivot(index='fold',columns='seed',values='score')\n\ndf_results.loc['seed_mean']= df_results.mean(numeric_only=True, axis=0)\ndf_results.loc[:,'fold_mean'] = df_results.mean(numeric_only=True, axis=1)\ndf_results","metadata":{"papermill":{"duration":0.254835,"end_time":"2022-01-02T19:39:47.079815","exception":false,"start_time":"2022-01-02T19:39:46.82498","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_importance(importances, features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()\n    \nplot_importance(np.array(importances),train_data1.columns, PLOT_TOP_N = 20, figsize=(10, 20))","metadata":{"papermill":{"duration":0.229805,"end_time":"2022-01-02T19:39:48.026186","exception":false,"start_time":"2022-01-02T19:39:47.796381","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_importance_groups(importances, features_names = train_data1.columns, PLOT_TOP_N = 20, figsize=(4, 8)):\n    importance_df = pd.DataFrame(data=importances, columns=features_names)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    \n    t = sorted_importance_df.transpose()\n    t['groups'] = [s.split('_')[-1] for s in sorted_importance_df.columns]\n    \n    t = t.groupby('groups').sum().transpose()\n    t = t.loc[:,t.columns.isin(['last','max','mean','min','std'])]\n\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    #ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=t,\n                orient='h',\n                ax=ax)\n    plt.show()\n    \nplot_importance_groups(np.array(importances))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"indeed 'last' features play a major role. ","metadata":{}},{"cell_type":"code","source":"del train_data1, train_labels1, X_tr, X_va, y_tr, y_va ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission\n","metadata":{}},{"cell_type":"code","source":"test_data = pd.read_pickle('../input/amex-feature-engineering/test_data_agg.pkl').astype('float16')\n\nmissing_cols = [f for f in Features if f not in test_data.columns]\ntest_data[missing_cols] = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://stackoverflow.com/questions/2130016/splitting-a-list-into-n-parts-of-approximately-equal-length\ndef split(a, n):\n    k, m = divmod(len(a), n)\n    return (a[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(n))\n\nsplit_ids = split(test_data.index.unique(),10)\n\ndf_list_preds = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_sub = []\n\nfor (i,ids) in enumerate(split_ids):\n    print(f'chunk {i}')\n    test_data_ids = test_data[test_data.index.isin(ids)]\n    preds_ids_sub = []\n    for k in models:\n        print(f'model key {k}')\n        preds_ids_sub.append(models[k].predict_proba(test_data_ids, raw_score=True))\n    preds_sub.append(np.nanmean(np.array(preds_ids_sub),axis=0))\n    gc.collect()\n    \npreds_sub = np.hstack(preds_sub)\npreds_series = pd.Series(preds_sub,index = test_data.index.unique())\nproba_series = np.exp(preds_series)/(1+np.exp(preds_series))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_series.hist(bins=100)\nplt.axvline(x=np.log(0.04/(1-0.04)),color='black');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"proba_series.hist(bins=100)\nplt.axvline(x=0.04,color='black');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\ndf_sub.prediction = proba_series.loc[df_sub.customer_ID].values\n\ndf_sub = df_sub.set_index('customer_ID')\ndf_sub.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}