{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Baseline model - XGBoost\n\nAiming to perform some naive prediction techniques to understand how the classification challenge will work.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"Taken inspiration from @ambrosm [AMEX EDA which makes sense](https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense). Analysis has also been sourced from @cdeotte [XGBoost starter](https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793/notebook).","metadata":{}},{"cell_type":"markdown","source":"### Objectives\n* Build baseline model\n* Take a reduced dataset with training variables\n* Add the competition evaluation metric to the XGBoost model","metadata":{}},{"cell_type":"markdown","source":"## Updates\n* eval_metric: making use of the aucpr (Area under the Precision Recall curve)","metadata":{}},{"cell_type":"code","source":"# Import packages\nimport pandas as pd\nimport numpy as np\nimport pickle\nfrom matplotlib import pyplot as plt\nimport os, gc\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import accuracy_score\nfrom xgboost import plot_importance\n\n# Import xgb modules\nimport xgboost as xgb\n\nimport cupy, cudf # GPU libraries","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-06-12T09:37:37.282248Z","iopub.execute_input":"2022-06-12T09:37:37.283154Z","iopub.status.idle":"2022-06-12T09:37:42.653583Z","shell.execute_reply.started":"2022-06-12T09:37:37.283035Z","shell.execute_reply":"2022-06-12T09:37:42.652552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('RAPIDS version',cudf.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:38:01.755169Z","iopub.execute_input":"2022-06-12T09:38:01.755838Z","iopub.status.idle":"2022-06-12T09:38:01.762660Z","shell.execute_reply.started":"2022-06-12T09:38:01.755803Z","shell.execute_reply":"2022-06-12T09:38:01.761360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The data\n\nThe dataset of this competition has a considerable size. If you read the original csv files, the data barely fits into memory. That's why we read the data from @munumbutt's [AMEX-Feather-Dataset](https://www.kaggle.com/datasets/munumbutt/amexfeather). In this [Feather](https://arrow.apache.org/docs/python/feather.html) file, the floating point precision has been reduced from 64 bit to 16 bit. And reading a Feather file is faster than reading a csv file because the Feather file format is binary.\n\nThere are 5.5 million rows for training and 11 million rows of test data.","metadata":{}},{"cell_type":"code","source":"# Original import strategy\n# %%time\n# train = pd.read_feather('../input/amexfeather/train_data.ftr')\n# test = pd.read_feather('../input/amexfeather/test_data.ftr')\n# with pd.option_context(\"display.min_rows\", 6):\n#     display(train)\n#     display(test)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T11:03:24.037385Z","iopub.execute_input":"2022-06-07T11:03:24.037733Z","iopub.status.idle":"2022-06-07T11:03:24.042811Z","shell.execute_reply.started":"2022-06-07T11:03:24.037709Z","shell.execute_reply":"2022-06-07T11:03:24.041527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FILL NAN VALUE\n# NAN_VALUE = -99 # will fit in int8","metadata":{"execution":{"iopub.status.busy":"2022-06-07T11:03:24.049228Z","iopub.execute_input":"2022-06-07T11:03:24.049778Z","iopub.status.idle":"2022-06-07T11:03:24.053163Z","shell.execute_reply.started":"2022-06-07T11:03:24.049751Z","shell.execute_reply":"2022-06-07T11:03:24.052225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Updated code to include garbage collection to help release memory after data processing\n# Making use of the GPU library. This only works for integer only features at present.\ndef read_file_int(path = '', usecols = None):\n    # LOAD DATAFRAME\n    if usecols is not None: df = cudf.read_feather(path, columns=usecols)\n    else: df = cudf.read_feather(path)\n    # REDUCE DTYPE FOR CUSTOMER AND DATE\n#   df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    df.S_2 = cudf.to_datetime(df.S_2)\n    # CREATE OVERALL ROW MISS VALUE\n    features = [x for x in df.columns.values if x not in ['customer_ID', 'target']]\n    df['n_missing'] = df[features].isna().sum(axis=1)\n    # FILL NAN\n    df = df.fillna(NAN_VALUE) \n    # KEEP ONLY FINAL CUSTOMER ID UNTIL FUTURE TIME SERIES WORK BEGINS\n    df_out = df.groupby(['customer_ID']).nth(-1).reset_index(drop=True)\n    print('shape of data:', df_out.shape)\n    del df\n    _ = gc.collect()\n    return df_out\n\n# To ensure that the categorical features are imported only using CPU\ndef read_file_cpu(path = '', usecols = None):\n    # LOAD DATAFRAME\n    if usecols is not None: df = pd.read_feather(path, columns=usecols)\n    else: df = pd.read_feather(path)\n    # REDUCE DTYPE FOR CUSTOMER AND DATE\n#   df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    df.S_2 = pd.to_datetime(df.S_2)\n    # CREATE OVERALL ROW MISS VALUE\n    features = [x for x in df.columns.values if x not in ['customer_ID', 'target']]\n    df['n_missing'] = df[features].isna().sum(axis=1)\n    # FILL NAN\n#     features_num = [x for x in df._get_numeric_data().columns.values if x not in ['customer_ID', 'target']]\n#     df = df[features_num].fillna(NAN_VALUE) \n    # KEEP ONLY FINAL CUSTOMER ID UNTIL FUTURE TIME SERIES WORK BEGINS\n    df_out = df.groupby(['customer_ID']).nth(-1).reset_index(drop=True)\n    print('shape of data:', df_out.shape)\n    del df\n    _ = gc.collect()\n    return df_out","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:39:11.423674Z","iopub.execute_input":"2022-06-12T09:39:11.424084Z","iopub.status.idle":"2022-06-12T09:39:11.439689Z","shell.execute_reply.started":"2022-06-12T09:39:11.424053Z","shell.execute_reply":"2022-06-12T09:39:11.438527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Reading train data...')\nTRAIN_PATH = '../input/amexfeather/train_data.ftr'\ntrain_df = read_file_cpu(path = TRAIN_PATH)\n\nprint('Reading test data...')\nTEST_PATH = '../input/amexfeather/test_data.ftr'\ntest_df = read_file_cpu(path = TEST_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:39:19.922432Z","iopub.execute_input":"2022-06-12T09:39:19.922872Z","iopub.status.idle":"2022-06-12T09:41:15.219966Z","shell.execute_reply.started":"2022-06-12T09:39:19.922843Z","shell.execute_reply":"2022-06-12T09:41:15.213440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The target column of the train dataframe corresponds to the target column of train_labels.csv. In the csv file of the train data, there is no target column; it has been joined into the Feather file as a convenience.\n\nS_2 is the statement date. All train statement dates are between March of 2017 and March of 2018 (13 months), and no statement dates are missing. All test statement dates are between April of 2018 and October of 2019. This means that the statement dates of train and test don't overlap:","metadata":{}},{"cell_type":"code","source":"# Understanding the file size of one file\nfrom humanize import naturalsize\nsize = train_df.memory_usage(deep='True').sum()\nprint(size)\nprint(naturalsize(size))","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:42:16.825112Z","iopub.execute_input":"2022-06-12T09:42:16.825604Z","iopub.status.idle":"2022-06-12T09:42:16.869964Z","shell.execute_reply.started":"2022-06-12T09:42:16.825567Z","shell.execute_reply":"2022-06-12T09:42:16.868874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train data memory usage: {naturalsize(train_df.memory_usage(deep=\"True\").sum())} ')\nprint(f'Test data memory usage:  {naturalsize(test_df.memory_usage(deep=\"True\").sum())}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:42:18.149215Z","iopub.execute_input":"2022-06-12T09:42:18.149753Z","iopub.status.idle":"2022-06-12T09:42:18.176804Z","shell.execute_reply.started":"2022-06-12T09:42:18.149706Z","shell.execute_reply":"2022-06-12T09:42:18.175563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Analysis","metadata":{}},{"cell_type":"markdown","source":"Removing many of the highly correlated features resulted in a reduction in model performance. For the time being the features will be retained. However, more analysis is required to understand if this feature reduction makes sense.","metadata":{}},{"cell_type":"code","source":"# Correlation matrix\ncorr = train_df.corr()\n# Mask the upper triangle\nmask = np.triu(np.ones_like(corr, dtype=bool))\n# Add the mask to the heatmap\nfig, ax = plt.subplots(1, 1, figsize=(20,14))\nsns.heatmap(corr, mask=mask, center=0, linewidths=1, annot=True, fmt=\".2f\", ax=ax)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:42:25.160770Z","iopub.execute_input":"2022-06-12T09:42:25.161203Z","iopub.status.idle":"2022-06-12T09:44:05.987114Z","shell.execute_reply.started":"2022-06-12T09:42:25.161170Z","shell.execute_reply":"2022-06-12T09:44:05.985832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove highly correlated features\ncorr_matrix = corr.abs()\n# Create a boolean mask and apply it\nmask = np.triu(np.ones_like(corr_matrix, dtype=bool))\ntri_df = corr_matrix.mask(mask)\n\n# List column names of highly correlated features (r > 0.7)\nto_drop = [c for c in tri_df.columns if any(tri_df[c] > 0.7)]\nprint(f'Number of features: {len(to_drop)} \\n {to_drop}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:44:32.241248Z","iopub.execute_input":"2022-06-12T09:44:32.241880Z","iopub.status.idle":"2022-06-12T09:44:32.345594Z","shell.execute_reply.started":"2022-06-12T09:44:32.241828Z","shell.execute_reply":"2022-06-12T09:44:32.344034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove the highly correlated features\n# train_df = train_df.drop(to_drop, axis=1)\n# train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-07T11:12:42.023701Z","iopub.execute_input":"2022-06-07T11:12:42.024214Z","iopub.status.idle":"2022-06-07T11:12:42.380485Z","shell.execute_reply.started":"2022-06-07T11:12:42.024169Z","shell.execute_reply":"2022-06-07T11:12:42.379485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset prepared for analysis","metadata":{}},{"cell_type":"code","source":"target = train_df['target']\ntrain_df = train_df.drop(['target'], axis=1)\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:44:39.344924Z","iopub.execute_input":"2022-06-12T09:44:39.345705Z","iopub.status.idle":"2022-06-12T09:44:39.677991Z","shell.execute_reply.started":"2022-06-12T09:44:39.345654Z","shell.execute_reply":"2022-06-12T09:44:39.676875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Competition metric performance","metadata":{}},{"cell_type":"markdown","source":"The numpy metric for evaluation has been taken from @rohanrao [AMEX: Competition Metric Implementations](https://www.kaggle.com/code/rohanrao/amex-competition-metric-implementations)","metadata":{}},{"cell_type":"code","source":"def amex_metric_numpy(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = np.sum(y_true)\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by describing prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_mask = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = np.sum(target[four_pct_mask]) / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:44:55.604188Z","iopub.execute_input":"2022-06-12T09:44:55.604650Z","iopub.status.idle":"2022-06-12T09:44:55.616620Z","shell.execute_reply.started":"2022-06-12T09:44:55.604618Z","shell.execute_reply":"2022-06-12T09:44:55.615349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building baseline","metadata":{}},{"cell_type":"markdown","source":"This model should help to provide an early building block of what to expect with the challenge. Lets try to review only the last entry by each customer_ID for initial model discovery","metadata":{}},{"cell_type":"code","source":"num_features = train_df._get_numeric_data().columns\nnum_features","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:45:07.582147Z","iopub.execute_input":"2022-06-12T09:45:07.582831Z","iopub.status.idle":"2022-06-12T09:45:07.591358Z","shell.execute_reply.started":"2022-06-12T09:45:07.582796Z","shell.execute_reply":"2022-06-12T09:45:07.589863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the arrays for features and the target: X, y\nX, y = train_df._get_numeric_data(), target","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:45:09.291468Z","iopub.execute_input":"2022-06-12T09:45:09.291861Z","iopub.status.idle":"2022-06-12T09:45:09.298443Z","shell.execute_reply.started":"2022-06-12T09:45:09.291831Z","shell.execute_reply":"2022-06-12T09:45:09.296094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the training and test datasets\nX_train, X_test, y_train, y_test = train_test_split(X, \n                                                    y, \n                                                    test_size = 0.2, \n                                                    random_state=100,\n                                                    stratify=y)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:45:14.239556Z","iopub.execute_input":"2022-06-12T09:45:14.240127Z","iopub.status.idle":"2022-06-12T09:45:15.664719Z","shell.execute_reply.started":"2022-06-12T09:45:14.240081Z","shell.execute_reply":"2022-06-12T09:45:15.663397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate the classifier. Can switch on parameter tree_method='gpu_hist' in the future\nxg_cl = XGBClassifier(objective='binary:logistic', \n                      n_estimators=10,\n                      seed=123,\n                      use_label_encoder=False,\n                      eval_metric='aucpr', # updated to make use of the aucpr option\n                      early_stopping_rounds=10,\n                      tree_method='gpu_hist',\n                      enable_categorical=True\n                      )\neval_set = [(X_test, y_test)]","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:46:42.893744Z","iopub.execute_input":"2022-06-12T09:46:42.894160Z","iopub.status.idle":"2022-06-12T09:46:42.901416Z","shell.execute_reply.started":"2022-06-12T09:46:42.894128Z","shell.execute_reply":"2022-06-12T09:46:42.899219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the classifier\nxg_cl.fit(X_train, y_train, eval_set=eval_set, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:46:48.838196Z","iopub.execute_input":"2022-06-12T09:46:48.838691Z","iopub.status.idle":"2022-06-12T09:46:54.006276Z","shell.execute_reply.started":"2022-06-12T09:46:48.838658Z","shell.execute_reply":"2022-06-12T09:46:54.005281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict the labels of the test set\npreds = xg_cl.predict(X_test)\npreds_prob = xg_cl.predict_proba(X_test)[:,1]\n\n# Compute accuracy\naccuracy = accuracy_score(y_test, preds)\nprint(f'accuracy: {accuracy: .2%}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:47:02.212301Z","iopub.execute_input":"2022-06-12T09:47:02.212761Z","iopub.status.idle":"2022-06-12T09:47:02.862299Z","shell.execute_reply.started":"2022-06-12T09:47:02.212728Z","shell.execute_reply":"2022-06-12T09:47:02.861176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Review the important features\n# print(xg_cl.feature_importances_)\ndef plot_features(booster, figsize, max_num_features=15):    \n    fig, ax = plt.subplots(1,1,figsize=figsize)\n    return plot_importance(booster=booster, ax=ax, max_num_features=max_num_features)\nplot_features(xg_cl, (10,14))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:47:06.313803Z","iopub.execute_input":"2022-06-12T09:47:06.314374Z","iopub.status.idle":"2022-06-12T09:47:06.811246Z","shell.execute_reply.started":"2022-06-12T09:47:06.314290Z","shell.execute_reply":"2022-06-12T09:47:06.810036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Metric Evaluation Values\\n')\nprint(f'Numpy: {amex_metric_numpy(y_test.to_numpy().ravel(), preds_prob)}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:47:11.262824Z","iopub.execute_input":"2022-06-12T09:47:11.263597Z","iopub.status.idle":"2022-06-12T09:47:11.310593Z","shell.execute_reply.started":"2022-06-12T09:47:11.263558Z","shell.execute_reply":"2022-06-12T09:47:11.309356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boosting and CV methods\nLets make use of the boosting and inbuilt CV methods","metadata":{}},{"cell_type":"code","source":"# Understanding weighted class imbalance\nfrom collections import Counter\n\ncounter = Counter(y)\nprint(counter)\n\n# estimate scale_pos_weight value\nestimate = counter[0] / counter[1]\nprint('Estimate: %.3f' % estimate)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:47:18.684712Z","iopub.execute_input":"2022-06-12T09:47:18.685138Z","iopub.status.idle":"2022-06-12T09:47:18.778270Z","shell.execute_reply.started":"2022-06-12T09:47:18.685104Z","shell.execute_reply":"2022-06-12T09:47:18.777075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the DMatrix from X and y: churn_dmatrix\nd_train = xgb.DMatrix(data=X_train, label=y_train)\nd_test = xgb.DMatrix(data=X_test, label=y_test)\nxgd_test = xgb.DMatrix(data=test_df._get_numeric_data())\n\n# Create the parameter dictionary: params. NOTE: have to explicitly provide the objective param\nparams = {\"objective\":\"binary:logistic\", \n          \"max_depth\": 6,\n          \"eval_metric\":'aucpr', # updated to make use of the aucpr option\n          \"tree_method\":'gpu_hist',\n          \"predictor\": 'gpu_predictor',\n#           \"scale_pos_weight\": 30,\n         }\n\n# Reviewing the AUC metric\n# Perform cross_validation: cv_results\ncv_results = xgb.cv(dtrain=d_train, params=params,\n                    nfold=5, num_boost_round=10, \n                    metrics=\"aucpr\", as_pandas=True, seed=123)\n\n# Print cv_results\nprint(cv_results)\n\n# Print the AUC\n# print((cv_results[\"test-auc-mean\"]).iloc[-1])\nprint((cv_results[\"test-aucpr-mean\"]).iloc[-1])","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:49:36.896227Z","iopub.execute_input":"2022-06-12T09:49:36.896699Z","iopub.status.idle":"2022-06-12T09:49:53.914978Z","shell.execute_reply.started":"2022-06-12T09:49:36.896666Z","shell.execute_reply":"2022-06-12T09:49:53.913802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Review the train method\nparams = {\n    \"objective\":\"binary:logistic\", \n    \"max_depth\": 6,\n    \"eval_metric\":'aucpr', \n    \"tree_method\":'gpu_hist',\n    \"predictor\": 'gpu_predictor',\n#     \"scale_pos_weight\": 30,    \n}\n\n# train - verbose_eval option switches off the log outputs\nxgb_clf = xgb.train(\n    params,\n    d_train,\n    num_boost_round=5000,\n    evals=[(d_train, 'train'), (d_test, 'test')],\n    early_stopping_rounds=10,\n    verbose_eval=0\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:02.711099Z","iopub.execute_input":"2022-06-12T09:50:02.711571Z","iopub.status.idle":"2022-06-12T09:50:04.541324Z","shell.execute_reply.started":"2022-06-12T09:50:02.711534Z","shell.execute_reply":"2022-06-12T09:50:04.540150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict\ny_pred = xgb_clf.predict(d_test)\n\n# Compute and print metrics\nprint('Metric Evaluation Values\\n')\nprint(f'Numpy: {amex_metric_numpy(y_test.to_numpy().ravel(), y_pred)}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:10.065145Z","iopub.execute_input":"2022-06-12T09:50:10.065558Z","iopub.status.idle":"2022-06-12T09:50:10.136083Z","shell.execute_reply.started":"2022-06-12T09:50:10.065524Z","shell.execute_reply":"2022-06-12T09:50:10.134650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The booster has helped to improve the model performance. Will try to add the custom objective for this challenge","metadata":{}},{"cell_type":"markdown","source":"## Rank Order table","metadata":{}},{"cell_type":"markdown","source":"Evaluating model performance for rank ordering tasks. Taking guidance from [7 Important Model Performance Measures](https://www.k2analytics.co.in/7-important-model-performance-measures/#:~:text=Rank%20Order%20Table%20is%20a,from%20Non%2DChurners%2C%20etc.)","metadata":{}},{"cell_type":"code","source":"train_df._get_numeric_data().columns","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:20.563716Z","iopub.execute_input":"2022-06-12T09:50:20.564146Z","iopub.status.idle":"2022-06-12T09:50:20.576393Z","shell.execute_reply.started":"2022-06-12T09:50:20.564113Z","shell.execute_reply":"2022-06-12T09:50:20.574880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets build using the X_test data - this was to check and see if the code worked. Now going to score up the train_df to get a larger sample\n# 1. Predict probability\n# rank_data = X_test.copy()\n# rank_data['target'] = y_test\n# rank_data['prob'] = preds_prob\n# rank_data.head()\n\nrank_data = train_df._get_numeric_data()\nxgd_rank = xgb.DMatrix(data=train_df._get_numeric_data())\nrank_data['prob'] = xgb_clf.predict(xgd_rank)\nrank_data['target'] = target\nrank_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:22.974872Z","iopub.execute_input":"2022-06-12T09:50:22.975261Z","iopub.status.idle":"2022-06-12T09:50:27.916505Z","shell.execute_reply.started":"2022-06-12T09:50:22.975229Z","shell.execute_reply":"2022-06-12T09:50:27.915253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First create the decile value by prob\nrank = rank_data.loc[:, ['target', 'prob']]\nrank[\"ranks\"] = rank['prob'].rank(method=\"first\")\n\n# The notes displayed here had related to only using the X_test dataframe. With the train_df being used we can try using the probabilities again\n# First method bunchs the final three buckets into one as there are a low of low probs\nrank['decile'] = pd.qcut(rank.prob, 10, labels=False, duplicates='drop') \n# Second method aims to use the rank method, however the nature of this rank is still random\n# An alternative for this piece might be to put the 'prob' in order and sort by the target\n# rank['decile'] = pd.qcut(rank.ranks, 10, labels=False)\n# Reviewing the lowest probability\nmin_prob = np.min(rank.prob)\nrank.loc[(rank.prob == min_prob)].head()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:33.492248Z","iopub.execute_input":"2022-06-12T09:50:33.492707Z","iopub.status.idle":"2022-06-12T09:50:33.673702Z","shell.execute_reply.started":"2022-06-12T09:50:33.492675Z","shell.execute_reply":"2022-06-12T09:50:33.672225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a rank_order table\ndef rank_order(df: pd.DataFrame, y: str, target: str) -> pd.DataFrame:\n    \n    rank = df.groupby('decile').apply(lambda x: pd.Series([\n        np.min(x[y]),\n        np.max(x[y]),\n        np.mean(x[y]),\n        np.size(x[y]),\n        np.sum(x[target]),\n        np.size(x[target][x[target]==0]),\n    ],\n        index=([\"min_prob\",\"max_prob\",\"avg_prob\",\n               \"cnt_cust\",\"cnt_def\",\"cnt_non_def\"])\n    )).reset_index()\n    rank = rank.sort_values(by='decile', ascending=False)\n    rank[\"drate\"] = round(rank[\"cnt_def\"]*100/rank[\"cnt_cust\"], 2)\n    rank[\"cum_cust\"] = np.cumsum(rank[\"cnt_cust\"])\n    rank[\"cum_def\"] = np.cumsum(rank[\"cnt_def\"])\n    rank[\"cum_non_def\"] = np.cumsum(rank[\"cnt_non_def\"])\n    rank[\"cum_cust_pct\"] = round(rank[\"cum_cust\"]*100/np.sum(rank[\"cnt_cust\"]), 2)\n    rank[\"cum_def_pct\"] = round(rank[\"cum_def\"]*100/np.sum(rank[\"cnt_def\"]), 2)\n    rank[\"cum_non_def_pct\"] = round(rank[\"cum_non_def\"]*100/np.sum(rank[\"cnt_non_def\"]), 2)\n    rank[\"KS\"] = round(rank[\"cum_def_pct\"] - rank[\"cum_non_def_pct\"],2)\n    rank[\"Lift\"] = round(rank[\"cum_def_pct\"] / rank[\"cum_non_def_pct\"],2)\n    return rank\n\nrank_gains_table = rank_order(rank, \"prob\", \"target\")\nrank_gains_table","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:36.475933Z","iopub.execute_input":"2022-06-12T09:50:36.476397Z","iopub.status.idle":"2022-06-12T09:50:36.579447Z","shell.execute_reply.started":"2022-06-12T09:50:36.476340Z","shell.execute_reply":"2022-06-12T09:50:36.578123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make submission","metadata":{}},{"cell_type":"code","source":"# Score up the test dataset\ntest_preds = xgb_clf.predict(xgd_test)\ntest_preds.view()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:44.637569Z","iopub.execute_input":"2022-06-12T09:50:44.638033Z","iopub.status.idle":"2022-06-12T09:50:45.326551Z","shell.execute_reply.started":"2022-06-12T09:50:44.638000Z","shell.execute_reply":"2022-06-12T09:50:45.325538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make submission\nsub_data = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\nsub_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:47.392399Z","iopub.execute_input":"2022-06-12T09:50:47.392809Z","iopub.status.idle":"2022-06-12T09:50:49.151168Z","shell.execute_reply.started":"2022-06-12T09:50:47.392776Z","shell.execute_reply":"2022-06-12T09:50:49.149724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_data['prediction'] = test_preds\nsub_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:52.185369Z","iopub.execute_input":"2022-06-12T09:50:52.185782Z","iopub.status.idle":"2022-06-12T09:50:52.201548Z","shell.execute_reply.started":"2022-06-12T09:50:52.185750Z","shell.execute_reply":"2022-06-12T09:50:52.199485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submission file\nsub_data.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T09:50:53.657009Z","iopub.execute_input":"2022-06-12T09:50:53.657692Z","iopub.status.idle":"2022-06-12T09:50:58.136166Z","shell.execute_reply.started":"2022-06-12T09:50:53.657656Z","shell.execute_reply":"2022-06-12T09:50:58.135027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}