{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import libraries\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport seaborn as sns\n\nfrom sklearn.model_selection import KFold, GridSearchCV, ParameterGrid, train_test_split\n\nimport lightgbm as lgb  # LightGBM, fast.\nimport xgboost as xgb  # XGBoost\n\nfrom itertools import cycle\nimport time  # sleep()","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:52:26.641676Z","iopub.execute_input":"2022-06-11T01:52:26.643309Z","iopub.status.idle":"2022-06-11T01:52:29.648610Z","shell.execute_reply.started":"2022-06-11T01:52:26.643036Z","shell.execute_reply":"2022-06-11T01:52:29.647411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Global variables and Constants\n\nRANDOM_STATE = 22\npd.set_option('display.float_format', '{:.4f}'.format)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:52:29.651195Z","iopub.execute_input":"2022-06-11T01:52:29.651783Z","iopub.status.idle":"2022-06-11T01:52:29.657700Z","shell.execute_reply.started":"2022-06-11T01:52:29.651736Z","shell.execute_reply":"2022-06-11T01:52:29.656550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CustomUtil:\n#     def infer_by_knn(_cls):\n        ","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:52:29.659674Z","iopub.execute_input":"2022-06-11T01:52:29.660155Z","iopub.status.idle":"2022-06-11T01:52:29.672053Z","shell.execute_reply.started":"2022-06-11T01:52:29.660111Z","shell.execute_reply":"2022-06-11T01:52:29.670644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/inversion/amex-competition-metric-python\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n\n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename({'target': 'prediction'}, axis='columns')\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:52:29.677599Z","iopub.execute_input":"2022-06-11T01:52:29.678363Z","iopub.status.idle":"2022-06-11T01:52:29.698797Z","shell.execute_reply.started":"2022-06-11T01:52:29.678309Z","shell.execute_reply":"2022-06-11T01:52:29.697633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load data\n\n# This won't work. Out-of-memory exception occurred.\n# train_df = pd.read_csv('../input/amex-default-prediction/train_data.csv')\n# test_df = pd.read_csv('../input/amex-default-prediction/test_data.csv')\n\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/327400\ntrain_df = pd.read_feather('../input/amex-default-prediction-feather/train.feather')\ntest_df = pd.read_feather('../input/amex-default-prediction-feather/test.feather')\ntrain_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv', dtype={'target': 'category'})","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:52:29.700468Z","iopub.execute_input":"2022-06-11T01:52:29.701474Z","iopub.status.idle":"2022-06-11T01:53:29.706757Z","shell.execute_reply.started":"2022-06-11T01:52:29.701419Z","shell.execute_reply":"2022-06-11T01:53:29.705508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape, test_df.shape, train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:53:29.708482Z","iopub.execute_input":"2022-06-11T01:53:29.709187Z","iopub.status.idle":"2022-06-11T01:53:29.717908Z","shell.execute_reply.started":"2022-06-11T01:53:29.709138Z","shell.execute_reply":"2022-06-11T01:53:29.716865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# Cast types\ntrain_df['S_2'] = pd.to_datetime(train_df['S_2'])\ntest_df['S_2'] = pd.to_datetime(test_df['S_2'])\n\n# Categorical featues\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n# train_df = train_df.drop(columns=cat_cols)\ntrain_df = train_df.astype(dict(zip(cat_cols, cycle(['category']))))\ntest_df = test_df.astype(dict(zip(cat_cols, cycle(['category']))))\n\n# NaN cells\ntrain_df = train_df.dropna(axis='columns')","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:53:29.719125Z","iopub.execute_input":"2022-06-11T01:53:29.719643Z","iopub.status.idle":"2022-06-11T01:53:52.772868Z","shell.execute_reply.started":"2022-06-11T01:53:29.719608Z","shell.execute_reply":"2022-06-11T01:53:52.771798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorical_df = pd.concat([train_df[['customer_ID', 'S_2', *cat_cols]], test_df[['customer_ID', 'S_2', *cat_cols]]], axis='index')\n# categorical_df = pd.get_dummies(categorical_df, sparse=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:53:52.773957Z","iopub.execute_input":"2022-06-11T01:53:52.774302Z","iopub.status.idle":"2022-06-11T01:53:52.778983Z","shell.execute_reply.started":"2022-06-11T01:53:52.774273Z","shell.execute_reply":"2022-06-11T01:53:52.777988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = train_df.drop(columns=cat_cols).merge(categorical_df, how='left')\n# test_df = test_df.drop(columns=cat_cols).merge(categorical_df, how='left')","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:53:52.780505Z","iopub.execute_input":"2022-06-11T01:53:52.780904Z","iopub.status.idle":"2022-06-11T01:53:52.792608Z","shell.execute_reply.started":"2022-06-11T01:53:52.780870Z","shell.execute_reply":"2022-06-11T01:53:52.791437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df[train_df.columns]","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:53:52.796886Z","iopub.execute_input":"2022-06-11T01:53:52.797772Z","iopub.status.idle":"2022-06-11T01:54:04.008480Z","shell.execute_reply.started":"2022-06-11T01:53:52.797728Z","shell.execute_reply":"2022-06-11T01:54:04.007277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge train data with labels\ntrain_df = train_df.merge(train_labels, on='customer_ID', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:04.010010Z","iopub.execute_input":"2022-06-11T01:54:04.010595Z","iopub.status.idle":"2022-06-11T01:54:21.968794Z","shell.execute_reply.started":"2022-06-11T01:54:04.010539Z","shell.execute_reply":"2022-06-11T01:54:21.967649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only use latest record per customer_ID (for brevity)\ntrain_df = train_df.groupby('customer_ID').tail(1).reset_index(drop=True)\ntest_df = test_df.groupby('customer_ID').tail(1).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:21.970188Z","iopub.execute_input":"2022-06-11T01:54:21.970610Z","iopub.status.idle":"2022-06-11T01:54:28.417097Z","shell.execute_reply.started":"2022-06-11T01:54:21.970577Z","shell.execute_reply":"2022-06-11T01:54:28.415841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:28.418773Z","iopub.execute_input":"2022-06-11T01:54:28.419189Z","iopub.status.idle":"2022-06-11T01:54:28.547196Z","shell.execute_reply.started":"2022-06-11T01:54:28.419153Z","shell.execute_reply":"2022-06-11T01:54:28.546418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:28.548342Z","iopub.execute_input":"2022-06-11T01:54:28.549170Z","iopub.status.idle":"2022-06-11T01:54:28.786912Z","shell.execute_reply.started":"2022-06-11T01:54:28.549128Z","shell.execute_reply":"2022-06-11T01:54:28.785560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"target_col = 'target'\ndroppable_cols = ['customer_ID', 'S_2', target_col]","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:28.788371Z","iopub.execute_input":"2022-06-11T01:54:28.788756Z","iopub.status.idle":"2022-06-11T01:54:28.793979Z","shell.execute_reply.started":"2022-06-11T01:54:28.788720Z","shell.execute_reply":"2022-06-11T01:54:28.792891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LightGBM","metadata":{}},{"cell_type":"code","source":"rm submission_*.*","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:28.795190Z","iopub.execute_input":"2022-06-11T01:54:28.795643Z","iopub.status.idle":"2022-06-11T01:54:29.653849Z","shell.execute_reply.started":"2022-06-11T01:54:28.795607Z","shell.execute_reply":"2022-06-11T01:54:29.652228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data\nX_train = train_df.drop(columns=droppable_cols)\ny_train = train_df[target_col]\nX_test = test_df.drop(columns=droppable_cols, errors='ignore')\n\n# Modeling\nis_modeling = False\nif is_modeling:\n    param_grid = {'learning_rate': [0.015], 'n_estimators': [1_500], 'num_leaves': [500], 'objective': ['binary']}\n    kf = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)\n    model = GridSearchCV(estimator=lgb.LGBMClassifier(), param_grid=param_grid, verbose=3, cv=kf)\n    model.fit(X_train, y_train)\n\n    # Output result\n    print('* Best params :', model.best_params_, sep='\\n')\n    print('* Best score :', model.best_score_, sep='\\t')\n    \n    # Feature importances\n    feature_importance_df = pd.DataFrame({'column': X_train.columns, 'importance': model.feature_importances_})\n    plt.figure(figsize=(20, 10))\n    sns.barplot(data=feature_importance_df, x='importance', y='column')\n    plt.show()\nelse:\n    model = lgb.LGBMClassifier(**{'learning_rate': 0.015, 'n_estimators': 1500, 'num_leaves': 500, 'objective': ['binary']})\n    model.fit(X_train, y_train)\n\n# Calculate amex metric\nam = amex_metric(y_train.astype('int8').to_frame(), pd.DataFrame({'prediction': model.predict(X_train).astype('int8')}))\nprint('amex metric :', am)\n\n# Inference\nsubmission_df = pd.DataFrame({'customer_ID': test_df['customer_ID'], 'prediction': model.predict(X_test)})\nsubmission_df.to_csv('submission_lgbm.csv.zip', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:54:29.655504Z","iopub.execute_input":"2022-06-11T01:54:29.655882Z","iopub.status.idle":"2022-06-11T02:04:04.107308Z","shell.execute_reply.started":"2022-06-11T01:54:29.655847Z","shell.execute_reply":"2022-06-11T02:04:04.106296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## XGBoost","metadata":{}},{"cell_type":"code","source":"# https://xgboost.readthedocs.io/en/stable/parameter.html","metadata":{"execution":{"iopub.status.busy":"2022-06-11T02:04:04.108774Z","iopub.execute_input":"2022-06-11T02:04:04.109636Z","iopub.status.idle":"2022-06-11T02:04:04.113806Z","shell.execute_reply.started":"2022-06-11T02:04:04.109598Z","shell.execute_reply":"2022-06-11T02:04:04.113090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-11T02:04:04.114896Z","iopub.execute_input":"2022-06-11T02:04:04.115700Z","iopub.status.idle":"2022-06-11T02:04:04.314436Z","shell.execute_reply.started":"2022-06-11T02:04:04.115662Z","shell.execute_reply":"2022-06-11T02:04:04.312983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sleep for GC\ntime.sleep(120)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T02:04:04.316187Z","iopub.execute_input":"2022-06-11T02:04:04.317234Z","iopub.status.idle":"2022-06-11T02:06:04.429731Z","shell.execute_reply.started":"2022-06-11T02:04:04.317182Z","shell.execute_reply":"2022-06-11T02:06:04.428040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data\nX_train = train_df.drop(columns=droppable_cols)\ny_train = train_df[target_col].astype('int8')  # XGBClassifier requires integer type\nX_test = test_df.drop(columns=droppable_cols, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2022-06-11T02:06:04.431339Z","iopub.execute_input":"2022-06-11T02:06:04.431977Z","iopub.status.idle":"2022-06-11T02:06:04.640565Z","shell.execute_reply.started":"2022-06-11T02:06:04.431930Z","shell.execute_reply":"2022-06-11T02:06:04.639156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Modeling\n# is_modeling = False\n# if is_modeling:\n#     param_grid = {'learning_rate': [0.01, 0.015], 'max_depth': [50, 100], 'n_estimators': [50, 100]}\n#     scores = []\n    \n#     for param in ParameterGrid(param_grid):\n#         model = xgb.XGBClassifier(objective='binary:logistic', eval_metric='error', tree_method='approx',\n#                                   enable_categorical=True, random_state=RANDOM_STATE, verbosity=1, **param)\n#         model.fit(X_train, y_train)\n#         score = model.score(X_train, y_train)\n#         scores.append((param, score))\n#         print('param & score', param, score, sep='\\n')\n        \n#     # Output result\n#     score_best = min(scores, key=lambda s: s[1])\n#     print(*scores, sep='\\n')\n# #     print('* Best score :', model.best_score, sep='\\t')\n    \n#     # Feature importances\n#     feature_importance_df = pd.DataFrame({'column': model.feature_names_in_, 'importance': model.feature_importances_})\n#     plt.figure(figsize=(20, 10))\n#     sns.barplot(data=feature_importance_df, x='importance', y='column')\n#     plt.show()\n# else:\n#     params = {'learning_rate': 0.01, 'max_depth': 50, 'n_estimators': 100}\n#     model = xgb.XGBClassifier(objective='binary:logistic', eval_metric='error', tree_method='approx',\n#                               enable_categorical=True, random_state=RANDOM_STATE, verbosity=1, **params)\n#     model.fit(X_train, y_train)\n\n# # Inference\n# test_df['prediction'] = model.predict(X_test)\n# test_df[['customer_ID', 'prediction']].to_csv('submission_xgb.csv.zip', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-11T02:06:04.641986Z","iopub.execute_input":"2022-06-11T02:06:04.642375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}