{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**<h4>This notebook implements XGBoost on the set processed by this [notebook](https://www.kaggle.com/code/susnato/amex-data-preprocesing-feature-engineering). Some of its code is inspired from some notebooks mentioned below.</h4>**\n\n<h4>If you like this notebook then please upvote.</h4>","metadata":{}},{"cell_type":"markdown","source":"**ACKNOWLEDGEMENTS**\n\n* Some of the code related to Feature Importance is taken from here https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793\n\nPlease checkout these notebooks too.","metadata":{}},{"cell_type":"markdown","source":"**IMPORTS**","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport glob\nimport tqdm\nimport numpy as np\nimport pandas as pd\n\nSEED = 42\nnp.random.seed(SEED)\nos.environ['PYTHONHASHSEED'] = str(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:07:02.909328Z","iopub.execute_input":"2022-06-07T19:07:02.910066Z","iopub.status.idle":"2022-06-07T19:07:02.938374Z","shell.execute_reply.started":"2022-06-07T19:07:02.909649Z","shell.execute_reply":"2022-06-07T19:07:02.937668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**LOAD THE DATA**\n\nThe data used for training here is created by the notebook : [AMEX-Data Preprocesing & Feature Engineering](https://www.kaggle.com/code/susnato/amex-data-preprocesing-feature-engineering)  You can checkout the notebook for more details.","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ntrain_labels['customer_ID'] = train_labels['customer_ID'].apply(lambda x: int(x[-16:], 16)).astype(np.int64)\ntrain_labels = train_labels.set_axis(train_labels['customer_ID'])\ntrain_labels = train_labels.drop(['customer_ID'], axis=1)\n\ntrain_pkls = sorted(glob.glob('../input/amex-data-preprocesing-feature-engineering/train_data_*'))\ntest_pkls = sorted(glob.glob('../input/amex-data-preprocesing-feature-engineering/test_data_*'))\n\ntrain_df = pd.read_pickle(train_pkls[0]).astype(np.float32)\nprint(train_pkls[0])\nfor i in train_pkls[1:]:\n    print(i)\n    train_df = train_df.append(pd.read_pickle(i))\n    train_df = train_df.astype(np.float32)\n    gc.collect()\n    \ny = train_labels.loc[train_df.index.values].values.astype(np.int8)\ntrain_df = train_df.drop(['D_64_1', 'D_66_0', 'D_68_0'], axis=1).astype(np.float32)\nprint(train_df.shape, y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:07:02.940175Z","iopub.execute_input":"2022-06-07T19:07:02.940559Z","iopub.status.idle":"2022-06-07T19:07:43.557409Z","shell.execute_reply.started":"2022-06-07T19:07:02.940525Z","shell.execute_reply":"2022-06-07T19:07:43.556484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**XGBOOST**","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\ndef amex_metric(y_true, y_pred):\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\n\ndef create_model(seed):\n    return xgb.XGBClassifier(\n        n_estimators = 1500,#\n        max_depth = 4,#\n        learning_rate = 0.07, #\n        subsample = 0.9, #\n        colsample_bytree = 0.3, #\n        min_child_weight = 7, #\n        reg_alpha = 2, \n        reg_lambda = 0.5, \n        tree_method ='gpu_hist',\n        predictor = 'gpu_predictor',\n        eval_metric = amex_metric,\n        random_state = SEED)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:07:43.558708Z","iopub.execute_input":"2022-06-07T19:07:43.559208Z","iopub.status.idle":"2022-06-07T19:07:44.276128Z","shell.execute_reply.started":"2022-06-07T19:07:43.559169Z","shell.execute_reply":"2022-06-07T19:07:44.275078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAINING**\n\nIf I try to use KFold CV the notebook craches due to shortage of system memory so I preferred to go with 75%-25% Holdout set.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(train_df, y,\n                                                    stratify=y, \n                                                    test_size=0.20)\ndel train_df, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:07:44.278038Z","iopub.execute_input":"2022-06-07T19:07:44.278403Z","iopub.status.idle":"2022-06-07T19:07:49.825188Z","shell.execute_reply.started":"2022-06-07T19:07:44.278368Z","shell.execute_reply":"2022-06-07T19:07:49.824323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(SEED)\nmodel.fit(X_train, y_train,\n         eval_set=[(X_train, y_train), (X_val, y_val)],\n         #early_stopping_rounds=200,\n         verbose=50)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:07:49.826626Z","iopub.execute_input":"2022-06-07T19:07:49.826999Z","iopub.status.idle":"2022-06-07T19:22:34.745575Z","shell.execute_reply.started":"2022-06-07T19:07:49.826964Z","shell.execute_reply":"2022-06-07T19:22:34.744691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h4>Now let's plot the feature importance</h4>","metadata":{}},{"cell_type":"code","source":"feature_important = model.get_booster().get_score(importance_type='weight')\nkeys = list(feature_important.keys())\nvalues = list(feature_important.values())\n\ndata = pd.DataFrame(data=values, index=keys, columns=[\"score\"]).sort_values(by = \"score\", ascending=False)\ndata.nlargest(25, columns=\"score\").plot(kind='barh', figsize = (20,10)) ## plot top 40 features","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:22:34.747129Z","iopub.execute_input":"2022-06-07T19:22:34.747517Z","iopub.status.idle":"2022-06-07T19:22:35.347408Z","shell.execute_reply.started":"2022-06-07T19:22:34.747481Z","shell.execute_reply":"2022-06-07T19:22:35.346552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SUBMISSION**","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\nsubmission['customer_ID_encoded'] = train_labels['customer_ID'] = submission['customer_ID'].apply(lambda x: int(x[-16:], 16)).astype(np.int64)\nsubmission.set_axis(submission['customer_ID_encoded'], inplace=True)\nsubmission = submission.drop(['customer_ID_encoded'], axis=1)\nsubmission['prediction'] = submission['prediction'].astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:22:35.351678Z","iopub.execute_input":"2022-06-07T19:22:35.352372Z","iopub.status.idle":"2022-06-07T19:22:37.622135Z","shell.execute_reply.started":"2022-06-07T19:22:35.352305Z","shell.execute_reply":"2022-06-07T19:22:37.62131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ncustomer_ids_list = []\npreds_list = []\nfor t in test_pkls:\n    test_df = pd.read_pickle(t)\n    customer_ids = test_df.axes[0].values\n    customer_ids = submission.loc[customer_ids]['customer_ID'].values\n    customer_ids_list.extend(customer_ids)\n    preds = model.predict_proba(test_df)[:, 1]\n    preds_list.extend(preds)\n    gc.collect()\n\npreds_list = np.array(preds_list).reshape(-1, 1)\ncustomer_ids_list = np.array(customer_ids_list).reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:22:37.623476Z","iopub.execute_input":"2022-06-07T19:22:37.623869Z","iopub.status.idle":"2022-06-07T19:24:22.298064Z","shell.execute_reply.started":"2022-06-07T19:22:37.623833Z","shell.execute_reply":"2022-06-07T19:24:22.297072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame(data=np.concatenate([customer_ids_list, preds_list], axis=1), columns=['customer_ID', 'prediction'])\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:24:22.29964Z","iopub.execute_input":"2022-06-07T19:24:22.300019Z","iopub.status.idle":"2022-06-07T19:24:27.112561Z","shell.execute_reply.started":"2022-06-07T19:24:22.299982Z","shell.execute_reply":"2022-06-07T19:24:27.11166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**CONCLUSION**\n\n* The results can be drastically improved by using CV.\n* I am currently implementing xgb.core.DataIter and trying to reduce the system memory usage of the notebook by deleting many unwanted features to use CV.\n* The hyper-parameters can also be tuned to imrpove LB score.","metadata":{}}]}