{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{"papermill":{"duration":0.008056,"end_time":"2022-05-26T09:44:50.04991","exception":false,"start_time":"2022-05-26T09:44:50.041854","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom lightgbm import LGBMClassifier, early_stopping, log_evaluation\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OrdinalEncoder\n\nimport gc","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.016931,"end_time":"2022-05-26T09:44:50.079476","exception":false,"start_time":"2022-05-26T09:44:50.062545","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:49:46.481984Z","iopub.execute_input":"2022-05-26T12:49:46.482289Z","iopub.status.idle":"2022-05-26T12:49:49.653356Z","shell.execute_reply.started":"2022-05-26T12:49:46.482221Z","shell.execute_reply":"2022-05-26T12:49:49.652573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CONFIG:\n    random_state = 69420\n    kaggle = True\n    kaggle_path = '../input/amexfeather/'\n    local_path = ''","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:49:49.655963Z","iopub.execute_input":"2022-05-26T12:49:49.656539Z","iopub.status.idle":"2022-05-26T12:49:49.661187Z","shell.execute_reply.started":"2022-05-26T12:49:49.656501Z","shell.execute_reply":"2022-05-26T12:49:49.660465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\ntrain = pd.read_feather(CONFIG.kaggle_path+'train_data.ftr')\ntrain.head()","metadata":{"papermill":{"duration":19.599095,"end_time":"2022-05-26T09:45:09.683255","exception":false,"start_time":"2022-05-26T09:44:50.08416","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:49:49.662744Z","iopub.execute_input":"2022-05-26T12:49:49.663227Z","iopub.status.idle":"2022-05-26T12:50:07.785153Z","shell.execute_reply.started":"2022-05-26T12:49:49.663191Z","shell.execute_reply":"2022-05-26T12:50:07.784422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Keep the last statement month per customer\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/327094\ntrain =  (train\n            .groupby('customer_ID')\n            .tail(1)\n            .set_index('customer_ID', drop=True)\n            .sort_index()\n            .drop(['S_2'], axis='columns'))","metadata":{"papermill":{"duration":5.855527,"end_time":"2022-05-26T09:45:15.543592","exception":false,"start_time":"2022-05-26T09:45:09.688065","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:16.430329Z","iopub.execute_input":"2022-05-26T12:50:16.430892Z","iopub.status.idle":"2022-05-26T12:50:19.139784Z","shell.execute_reply.started":"2022-05-26T12:50:16.430856Z","shell.execute_reply":"2022-05-26T12:50:19.138918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:50:19.141466Z","iopub.execute_input":"2022-05-26T12:50:19.142004Z","iopub.status.idle":"2022-05-26T12:50:19.271519Z","shell.execute_reply.started":"2022-05-26T12:50:19.141967Z","shell.execute_reply":"2022-05-26T12:50:19.270551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"papermill":{"duration":0.01562,"end_time":"2022-05-26T09:45:15.563932","exception":false,"start_time":"2022-05-26T09:45:15.548312","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:19.272899Z","iopub.execute_input":"2022-05-26T12:50:19.273370Z","iopub.status.idle":"2022-05-26T12:50:19.282134Z","shell.execute_reply.started":"2022-05-26T12:50:19.273335Z","shell.execute_reply":"2022-05-26T12:50:19.281260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_cols = train.columns.to_list()\n\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\nnum_cols = [col for col in all_cols if col not in cat_cols + [\"target\"]]","metadata":{"papermill":{"duration":0.012313,"end_time":"2022-05-26T09:45:15.581187","exception":false,"start_time":"2022-05-26T09:45:15.568874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:19.284033Z","iopub.execute_input":"2022-05-26T12:50:19.284381Z","iopub.status.idle":"2022-05-26T12:50:19.291276Z","shell.execute_reply.started":"2022-05-26T12:50:19.284347Z","shell.execute_reply":"2022-05-26T12:50:19.290276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train[cat_cols + num_cols]\ny = train['target']\n\nX.shape, y.shape","metadata":{"papermill":{"duration":0.296942,"end_time":"2022-05-26T09:45:15.883055","exception":false,"start_time":"2022-05-26T09:45:15.586113","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:30.225903Z","iopub.execute_input":"2022-05-26T12:50:30.226242Z","iopub.status.idle":"2022-05-26T12:50:30.492285Z","shell.execute_reply.started":"2022-05-26T12:50:30.226214Z","shell.execute_reply":"2022-05-26T12:50:30.491519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nenc = OrdinalEncoder()\nX[cat_cols] = enc.fit_transform(X[cat_cols])\n_ = gc.collect()","metadata":{"papermill":{"duration":0.157381,"end_time":"2022-05-26T09:45:16.045455","exception":false,"start_time":"2022-05-26T09:45:15.888074","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:32.253853Z","iopub.execute_input":"2022-05-26T12:50:32.254208Z","iopub.status.idle":"2022-05-26T12:50:33.102978Z","shell.execute_reply.started":"2022-05-26T12:50:32.254180Z","shell.execute_reply":"2022-05-26T12:50:33.102164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Competition metric","metadata":{"papermill":{"duration":0.004652,"end_time":"2022-05-26T09:45:16.055441","exception":false,"start_time":"2022-05-26T09:45:16.050789","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"papermill":{"duration":0.019887,"end_time":"2022-05-26T09:45:16.080141","exception":false,"start_time":"2022-05-26T09:45:16.060254","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:36.862937Z","iopub.execute_input":"2022-05-26T12:50:36.863490Z","iopub.status.idle":"2022-05-26T12:50:36.875363Z","shell.execute_reply.started":"2022-05-26T12:50:36.863456Z","shell.execute_reply":"2022-05-26T12:50:36.874618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Training","metadata":{"papermill":{"duration":0.005124,"end_time":"2022-05-26T09:45:16.926989","exception":false,"start_time":"2022-05-26T09:45:16.921865","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, stratify=y)","metadata":{"papermill":{"duration":562.72489,"end_time":"2022-05-26T09:54:39.65701","exception":false,"start_time":"2022-05-26T09:45:16.93212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:37.431599Z","iopub.execute_input":"2022-05-26T12:50:37.432362Z","iopub.status.idle":"2022-05-26T12:50:38.757413Z","shell.execute_reply.started":"2022-05-26T12:50:37.432326Z","shell.execute_reply":"2022-05-26T12:50:38.756588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_test.shape, y_train.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:50:38.759115Z","iopub.execute_input":"2022-05-26T12:50:38.759479Z","iopub.status.idle":"2022-05-26T12:50:38.766770Z","shell.execute_reply.started":"2022-05-26T12:50:38.759443Z","shell.execute_reply":"2022-05-26T12:50:38.765854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = LGBMClassifier(\n    n_estimators=50000,\n    device='gpu',\n    random_state=CONFIG.random_state,\n    extra_trees=True\n)","metadata":{"papermill":{"duration":562.72489,"end_time":"2022-05-26T09:54:39.65701","exception":false,"start_time":"2022-05-26T09:45:16.93212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:38.768471Z","iopub.execute_input":"2022-05-26T12:50:38.768839Z","iopub.status.idle":"2022-05-26T12:50:38.776531Z","shell.execute_reply.started":"2022-05-26T12:50:38.768804Z","shell.execute_reply":"2022-05-26T12:50:38.775829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nclf.fit(\n    X_train, y_train, \n    eval_set=[(X_test,y_test)],\n    callbacks=[early_stopping(50), log_evaluation(0)]\n)","metadata":{"papermill":{"duration":562.72489,"end_time":"2022-05-26T09:54:39.65701","exception":false,"start_time":"2022-05-26T09:45:16.93212","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:50:38.780096Z","iopub.execute_input":"2022-05-26T12:50:38.780384Z","iopub.status.idle":"2022-05-26T12:51:13.093120Z","shell.execute_reply.started":"2022-05-26T12:50:38.780358Z","shell.execute_reply":"2022-05-26T12:51:13.092523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.DataFrame(y_test.copy(deep=True))\ny_pred = y_pred.rename(columns={'target':'prediction'})\ny_pred","metadata":{"papermill":{"duration":5.402034,"end_time":"2022-05-26T09:54:45.064882","exception":false,"start_time":"2022-05-26T09:54:39.662848","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:51:13.096307Z","iopub.execute_input":"2022-05-26T12:51:13.098135Z","iopub.status.idle":"2022-05-26T12:51:13.109813Z","shell.execute_reply.started":"2022-05-26T12:51:13.098092Z","shell.execute_reply":"2022-05-26T12:51:13.108951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ny_pred[\"prediction\"] = clf.predict_proba(X_test)[:,1]","metadata":{"papermill":{"duration":5.402034,"end_time":"2022-05-26T09:54:45.064882","exception":false,"start_time":"2022-05-26T09:54:39.662848","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:51:13.111054Z","iopub.execute_input":"2022-05-26T12:51:13.111467Z","iopub.status.idle":"2022-05-26T12:51:15.947845Z","shell.execute_reply.started":"2022-05-26T12:51:13.111430Z","shell.execute_reply":"2022-05-26T12:51:15.947123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:51:15.951588Z","iopub.execute_input":"2022-05-26T12:51:15.953222Z","iopub.status.idle":"2022-05-26T12:51:15.965445Z","shell.execute_reply.started":"2022-05-26T12:51:15.953189Z","shell.execute_reply":"2022-05-26T12:51:15.964546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = pd.DataFrame(y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:51:15.967104Z","iopub.execute_input":"2022-05-26T12:51:15.967733Z","iopub.status.idle":"2022-05-26T12:51:15.976369Z","shell.execute_reply.started":"2022-05-26T12:51:15.967698Z","shell.execute_reply":"2022-05-26T12:51:15.975591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\namex_metric(y_test, y_pred)","metadata":{"papermill":{"duration":0.380587,"end_time":"2022-05-26T09:54:45.451371","exception":false,"start_time":"2022-05-26T09:54:45.070784","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:51:15.977730Z","iopub.execute_input":"2022-05-26T12:51:15.978302Z","iopub.status.idle":"2022-05-26T12:51:16.328805Z","shell.execute_reply.started":"2022-05-26T12:51:15.978267Z","shell.execute_reply":"2022-05-26T12:51:16.328013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.006107,"end_time":"2022-05-26T09:54:45.463587","exception":false,"start_time":"2022-05-26T09:54:45.45748","status":"completed"},"tags":[]}},{"cell_type":"code","source":"del train, X, y, X_test, X_train, y_train, y_test, y_pred\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:51:16.330184Z","iopub.execute_input":"2022-05-26T12:51:16.330618Z","iopub.status.idle":"2022-05-26T12:51:16.502499Z","shell.execute_reply.started":"2022-05-26T12:51:16.330578Z","shell.execute_reply":"2022-05-26T12:51:16.501709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = pd.read_feather(CONFIG.kaggle_path+'test_data.ftr')\ntest.head()","metadata":{"papermill":{"duration":41.130525,"end_time":"2022-05-26T09:55:26.600165","exception":false,"start_time":"2022-05-26T09:54:45.46964","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:54:53.590723Z","iopub.execute_input":"2022-05-26T12:54:53.591084Z","iopub.status.idle":"2022-05-26T12:55:29.756328Z","shell.execute_reply.started":"2022-05-26T12:54:53.591056Z","shell.execute_reply":"2022-05-26T12:55:29.755520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test =  (\n    test\n    .groupby('customer_ID')\n    .tail(1)\n    .set_index('customer_ID', drop=True)\n    .sort_index()\n    .drop(['S_2'], axis='columns')\n)","metadata":{"papermill":{"duration":19.658725,"end_time":"2022-05-26T09:55:46.265123","exception":false,"start_time":"2022-05-26T09:55:26.606398","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:55:29.757887Z","iopub.execute_input":"2022-05-26T12:55:29.758700Z","iopub.status.idle":"2022-05-26T12:55:35.412094Z","shell.execute_reply.started":"2022-05-26T12:55:29.758645Z","shell.execute_reply":"2022-05-26T12:55:35.411224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-26T12:55:35.413502Z","iopub.execute_input":"2022-05-26T12:55:35.413879Z","iopub.status.idle":"2022-05-26T12:55:35.562812Z","shell.execute_reply.started":"2022-05-26T12:55:35.413844Z","shell.execute_reply":"2022-05-26T12:55:35.561765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest[cat_cols] = enc.transform(test[cat_cols])\n_ = gc.collect()","metadata":{"papermill":{"duration":0.218876,"end_time":"2022-05-26T09:55:46.492963","exception":false,"start_time":"2022-05-26T09:55:46.274087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:55:35.565407Z","iopub.execute_input":"2022-05-26T12:55:35.566124Z","iopub.status.idle":"2022-05-26T12:55:36.848049Z","shell.execute_reply.started":"2022-05-26T12:55:35.566084Z","shell.execute_reply":"2022-05-26T12:55:36.847104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"prediction\"] = clf.predict_proba(test[cat_cols + num_cols])[:,1]\ntest.head()","metadata":{"papermill":{"duration":43.71662,"end_time":"2022-05-26T09:56:30.215789","exception":false,"start_time":"2022-05-26T09:55:46.499169","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:55:36.850325Z","iopub.execute_input":"2022-05-26T12:55:36.850717Z","iopub.status.idle":"2022-05-26T12:56:02.493870Z","shell.execute_reply.started":"2022-05-26T12:55:36.850666Z","shell.execute_reply":"2022-05-26T12:56:02.492986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"prediction\"].to_csv(\"submission.csv\", index=True)","metadata":{"papermill":{"duration":2.793365,"end_time":"2022-05-26T09:56:33.015382","exception":false,"start_time":"2022-05-26T09:56:30.222017","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-26T12:56:02.495295Z","iopub.execute_input":"2022-05-26T12:56:02.499285Z","iopub.status.idle":"2022-05-26T12:56:07.223911Z","shell.execute_reply.started":"2022-05-26T12:56:02.499243Z","shell.execute_reply":"2022-05-26T12:56:07.223078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}