{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## **Import Necessary Library**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nfrom lightgbm import LGBMClassifier, early_stopping, log_evaluation\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:49:52.488844Z","iopub.execute_input":"2022-07-14T14:49:52.489531Z","iopub.status.idle":"2022-07-14T14:49:56.591171Z","shell.execute_reply.started":"2022-07-14T14:49:52.489453Z","shell.execute_reply":"2022-07-14T14:49:56.590318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CONFIG:\n    random_state = 4222\n    kaggle = True\n    path = '../input/amexfeather'\n    local_path = ''","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:49:56.605748Z","iopub.execute_input":"2022-07-14T14:49:56.611190Z","iopub.status.idle":"2022-07-14T14:49:56.622884Z","shell.execute_reply.started":"2022-07-14T14:49:56.611149Z","shell.execute_reply":"2022-07-14T14:49:56.620746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Data Preprocessing**","metadata":{}},{"cell_type":"code","source":"#%%time\n#train = pd.read_feather(CONFIG.path + '/train_data.ftr')\n\ntrain = pd.read_feather(f'{CONFIG.path}/train_data.ftr')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:49:56.634497Z","iopub.execute_input":"2022-07-14T14:49:56.637282Z","iopub.status.idle":"2022-07-14T14:50:15.821569Z","shell.execute_reply.started":"2022-07-14T14:49:56.637238Z","shell.execute_reply":"2022-07-14T14:50:15.820765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No customer falls into default within the 13-month period. \n\n#D = pd.DataFrame(train, columns =[\"customer_ID\", \"target\"])\n#D = D.loc[D[\"target\"] == 1]\n#D.head()\n#del train\n\n#yy = D.groupby(\"customer_ID\", as_index=False).sum()\n#ys = D.groupby(\"customer_ID\",as_index=False).size()\n\n#yy['target'].equals(ys['size'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select last statement per customer and make customer_ID the index\n\ntrain = train.groupby('customer_ID')\ntrain = train.tail(1)\ntrain = train.drop(['S_2'], axis=1)\ntrain.set_index('customer_ID', inplace=True) \ntrain.tail(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:29.323365Z","iopub.execute_input":"2022-07-14T14:50:29.324255Z","iopub.status.idle":"2022-07-14T14:50:31.916923Z","shell.execute_reply.started":"2022-07-14T14:50:29.324214Z","shell.execute_reply":"2022-07-14T14:50:31.916135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"reference: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327094","metadata":{}},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:42.144357Z","iopub.execute_input":"2022-07-14T14:50:42.144708Z","iopub.status.idle":"2022-07-14T14:50:42.274635Z","shell.execute_reply.started":"2022-07-14T14:50:42.144678Z","shell.execute_reply":"2022-07-14T14:50:42.273757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_cols = train.columns.to_list()\ncat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nnum_features = [col for col in total_cols if col not in cat_features + [\"target\", \"customer_ID\", \"S_2\"] ]","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:45.886178Z","iopub.execute_input":"2022-07-14T14:50:45.886726Z","iopub.status.idle":"2022-07-14T14:50:45.893403Z","shell.execute_reply.started":"2022-07-14T14:50:45.886681Z","shell.execute_reply":"2022-07-14T14:50:45.892332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(num_features) + len(cat_features)\nfeatures = num_features + cat_features","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:47.818100Z","iopub.execute_input":"2022-07-14T14:50:47.818471Z","iopub.status.idle":"2022-07-14T14:50:47.822578Z","shell.execute_reply.started":"2022-07-14T14:50:47.818440Z","shell.execute_reply":"2022-07-14T14:50:47.821547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = train[cat_features + num_features]\ny = train['target']\n\nx.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:51.745143Z","iopub.execute_input":"2022-07-14T14:50:51.745612Z","iopub.status.idle":"2022-07-14T14:50:52.171230Z","shell.execute_reply.started":"2022-07-14T14:50:51.745570Z","shell.execute_reply":"2022-07-14T14:50:52.170361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Apply OrdinalEncoder","metadata":{"execution":{"iopub.status.busy":"2022-06-24T10:37:48.21323Z","iopub.execute_input":"2022-06-24T10:37:48.213809Z","iopub.status.idle":"2022-06-24T10:37:48.217939Z","shell.execute_reply.started":"2022-06-24T10:37:48.213777Z","shell.execute_reply":"2022-06-24T10:37:48.216658Z"}}},{"cell_type":"code","source":"%%time\n\nenc = OrdinalEncoder()\nx[cat_features] = enc.fit_transform(x[cat_features])\n_ = gc.collect","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:50:55.413820Z","iopub.execute_input":"2022-07-14T14:50:55.414456Z","iopub.status.idle":"2022-07-14T14:50:56.192628Z","shell.execute_reply.started":"2022-07-14T14:50:55.414425Z","shell.execute_reply":"2022-07-14T14:50:56.191751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Competition Metric**","metadata":{}},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:01.017610Z","iopub.execute_input":"2022-07-14T14:51:01.017963Z","iopub.status.idle":"2022-07-14T14:51:01.031089Z","shell.execute_reply.started":"2022-07-14T14:51:01.017932Z","shell.execute_reply":"2022-07-14T14:51:01.030106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model Training**","metadata":{}},{"cell_type":"code","source":"# Next step: implement a K-fold CV method\n\nX_train, X_test, y_train, y_test = train_test_split(x,y,\n                            test_size=0.3,random_state=CONFIG.random_state, \n                                                    stratify = y)\nX_train.shape, X_test.shape, y_train.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:05.214372Z","iopub.execute_input":"2022-07-14T14:51:05.214867Z","iopub.status.idle":"2022-07-14T14:51:06.827666Z","shell.execute_reply.started":"2022-07-14T14:51:05.214825Z","shell.execute_reply":"2022-07-14T14:51:06.826895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import optuna\n#import optuna.integration.lightgbm as lgb\n#from sklearn.model_selection import RepeatedKFold\n\n#import warnings\n#warnings.simplefilter(action='ignore', category=FutureWarning)\n#warnings.simplefilter(action='ignore', category=UserWarning)\n\n#rkf = RepeatedKFold(n_splits = 3, n_repeats = 3, random_state=42)\n\n#fixed_params = {\n#    'objective': 'binary',\n#    'metric': 'auc',\n#    'boosting_type': 'gbdt',\n#    'force_row_wise' : True,\n#    'device':'gpu',\n#    'random_state' : CONFIG.random_state,\n#    'extra_trees' : True,\n#    'feature_pre_filter': False,\n#    'verbose' : -1,\n#    'n_estimators': 300,\n#    'early_stopping_round': 30\n#}\n\n#X = np.array(X_train[features])\n#y = np.array(y_train).flatten()\n\n#dtrain = lgb.Dataset(X, label = y, categorical_feature = 'auto')    \n\n#tuner = lgb.LightGBMTunerCV(\n#        fixed_params, dtrain, \n#        verbose_eval = None,\n#        time_budget = 60000,\n#        folds = rkf,\n#        #num_boost_round = 10,\n#        shuffle = True\n#)\n\n#tuner.run()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-11T02:42:09.234976Z","iopub.execute_input":"2022-07-11T02:42:09.235755Z","iopub.status.idle":"2022-07-11T02:42:09.240411Z","shell.execute_reply.started":"2022-07-11T02:42:09.235719Z","shell.execute_reply":"2022-07-11T02:42:09.239476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tuner.best_params","metadata":{"execution":{"iopub.status.busy":"2022-07-11T01:28:51.267077Z","iopub.execute_input":"2022-07-11T01:28:51.267469Z","iopub.status.idle":"2022-07-11T01:28:51.274076Z","shell.execute_reply.started":"2022-07-11T01:28:51.267435Z","shell.execute_reply":"2022-07-11T01:28:51.27306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seach parameters from Optuna hypertuning and hand waving on num_leaves\n\nsearch_params = { \n    'learning_rate' : 0.065,\n    'lambda_l1': 5.465073323789261,\n    'lambda_l2': 8.252315889074575,\n    'num_leaves': 220,\n    'feature_fraction': 0.62,\n    'bagging_fraction': 0.9898640166643957,\n    'bagging_freq': 3,\n    'min_child_samples': 100\n}\n\nfixed_params={\n    'objective': 'binary',\n    'metric': 'auc',\n    'boosting_type' : 'gbdt',\n    'force_row_wise' : True,\n    'device': 'gpu',\n    'random_state' : CONFIG.random_state,\n    'extra_trees' : True,\n    'feature_pre_filter': False,\n    'n_estimators': 300,\n    'early_stopping_round': 30\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:12.554453Z","iopub.execute_input":"2022-07-14T14:51:12.554791Z","iopub.status.idle":"2022-07-14T14:51:12.561201Z","shell.execute_reply.started":"2022-07-14T14:51:12.554763Z","shell.execute_reply":"2022-07-14T14:51:12.560121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LGBMClassifier(**fixed_params, **search_params)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:16.849584Z","iopub.execute_input":"2022-07-14T14:51:16.849932Z","iopub.status.idle":"2022-07-14T14:51:16.854270Z","shell.execute_reply.started":"2022-07-14T14:51:16.849902Z","shell.execute_reply":"2022-07-14T14:51:16.853354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.fit(\n    X_train, y_train, \n    eval_set=[(X_test,y_test)],\n    callbacks=[log_evaluation(100)]\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:51:18.844461Z","iopub.execute_input":"2022-07-14T14:51:18.845160Z","iopub.status.idle":"2022-07-14T14:52:17.319809Z","shell.execute_reply.started":"2022-07-14T14:51:18.845109Z","shell.execute_reply":"2022-07-14T14:52:17.318959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Check trained model w/ X_test, y_test, and amex_metric**","metadata":{}},{"cell_type":"code","source":"%%time\ny_pred = pd.DataFrame(y_test.copy(deep=True))\ny_pred = y_pred.rename(columns={'target':'prediction'})\n\ny_pred[\"prediction\"] = model.predict_proba(X_test)[:,1]\ny_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:17.321345Z","iopub.execute_input":"2022-07-14T14:52:17.322199Z","iopub.status.idle":"2022-07-14T14:52:25.084003Z","shell.execute_reply.started":"2022-07-14T14:52:17.322159Z","shell.execute_reply":"2022-07-14T14:52:25.083135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = pd.DataFrame(y_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:25.088411Z","iopub.execute_input":"2022-07-14T14:52:25.090584Z","iopub.status.idle":"2022-07-14T14:52:25.098087Z","shell.execute_reply.started":"2022-07-14T14:52:25.090545Z","shell.execute_reply":"2022-07-14T14:52:25.097091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\namex_metric(y_test, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:25.100662Z","iopub.execute_input":"2022-07-14T14:52:25.101346Z","iopub.status.idle":"2022-07-14T14:52:25.572769Z","shell.execute_reply.started":"2022-07-14T14:52:25.101310Z","shell.execute_reply":"2022-07-14T14:52:25.571958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Confusion matrix**","metadata":{}},{"cell_type":"code","source":"y_pred_actual = model.predict(X_test)\ncm = confusion_matrix(y_test, y_pred_actual)\ncmd = ConfusionMatrixDisplay(cm,display_labels=['Not','Default'])\ncmd.plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:25.574159Z","iopub.execute_input":"2022-07-14T14:52:25.575416Z","iopub.status.idle":"2022-07-14T14:52:32.787927Z","shell.execute_reply.started":"2022-07-14T14:52:25.575345Z","shell.execute_reply":"2022-07-14T14:52:32.786743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Submission**","metadata":{}},{"cell_type":"code","source":"del train, x, y, X_test, X_train, y_train, y_test, y_pred\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:32.793213Z","iopub.execute_input":"2022-07-14T14:52:32.796442Z","iopub.status.idle":"2022-07-14T14:52:33.023915Z","shell.execute_reply.started":"2022-07-14T14:52:32.796369Z","shell.execute_reply":"2022-07-14T14:52:33.023027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = pd.read_feather(f'{CONFIG.path}/test_data.ftr')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:52:33.025511Z","iopub.execute_input":"2022-07-14T14:52:33.026153Z","iopub.status.idle":"2022-07-14T14:53:13.056518Z","shell.execute_reply.started":"2022-07-14T14:52:33.026115Z","shell.execute_reply":"2022-07-14T14:53:13.055744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.groupby('customer_ID')\ntest = test.tail(1)\ntest = test.drop(['S_2'], axis=1)\ntest.set_index('customer_ID', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:53:13.058725Z","iopub.execute_input":"2022-07-14T14:53:13.059126Z","iopub.status.idle":"2022-07-14T14:53:18.166436Z","shell.execute_reply.started":"2022-07-14T14:53:13.059088Z","shell.execute_reply":"2022-07-14T14:53:18.165597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:53:18.167901Z","iopub.execute_input":"2022-07-14T14:53:18.168291Z","iopub.status.idle":"2022-07-14T14:53:18.299938Z","shell.execute_reply.started":"2022-07-14T14:53:18.168242Z","shell.execute_reply":"2022-07-14T14:53:18.298937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest[cat_features] = enc.transform(test[cat_features])\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:53:18.303000Z","iopub.execute_input":"2022-07-14T14:53:18.303785Z","iopub.status.idle":"2022-07-14T14:53:19.805957Z","shell.execute_reply.started":"2022-07-14T14:53:18.303746Z","shell.execute_reply":"2022-07-14T14:53:19.805119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"prediction\"] = model.predict_proba(test[cat_features + num_features])[:,1]\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:53:19.807187Z","iopub.execute_input":"2022-07-14T14:53:19.808008Z","iopub.status.idle":"2022-07-14T14:54:09.618632Z","shell.execute_reply.started":"2022-07-14T14:53:19.807965Z","shell.execute_reply":"2022-07-14T14:54:09.617894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"prediction\"].to_csv(\"submission.csv\", index=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:54:09.619843Z","iopub.execute_input":"2022-07-14T14:54:09.620825Z","iopub.status.idle":"2022-07-14T14:54:13.961177Z","shell.execute_reply.started":"2022-07-14T14:54:09.620782Z","shell.execute_reply":"2022-07-14T14:54:13.960340Z"},"trusted":true},"execution_count":null,"outputs":[]}]}