{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**<h4>This notebook implements XGBoost on the set processed by this [notebook](https://www.kaggle.com/code/susnato/amex-data-preprocesing-feature-engineering). Some of its code is inspired from some notebooks mentioned below.</h4>**\n\n","metadata":{}},{"cell_type":"markdown","source":"**IMPORTING PYTHON LIBRARIES**","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport glob\nimport tqdm\nimport numpy as np\nimport pandas as pd\n\nSEED = 42\nnp.random.seed(SEED)\nos.environ['PYTHONHASHSEED'] = str(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T21:44:19.041812Z","iopub.execute_input":"2022-06-17T21:44:19.042476Z","iopub.status.idle":"2022-06-17T21:44:19.070546Z","shell.execute_reply.started":"2022-06-17T21:44:19.042388Z","shell.execute_reply":"2022-06-17T21:44:19.069809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**LOAD THE DATA**\n\nThe data used for training here is created by the notebook : [AMEX-Data Preprocesing & Feature Engineering](https://www.kaggle.com/code/susnato/amex-data-preprocesing-feature-engineering)  You can checkout the notebook for more details.","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ntrain_labels['customer_ID'] = train_labels['customer_ID'].apply(lambda x: int(x[-16:], 16)).astype(np.int64)\ntrain_labels = train_labels.set_axis(train_labels['customer_ID'])\ntrain_labels = train_labels.drop(['customer_ID'], axis=1)\n\ntrain_pkls = sorted(glob.glob('../input/amex-data-preprocesing-feature-engineering/train_data_*'))\ntest_pkls = sorted(glob.glob('../input/amex-data-preprocesing-feature-engineering/test_data_*'))\n\ntrain_df = pd.read_pickle(train_pkls[0]).astype(np.float32)\nprint(train_pkls[0])\nfor i in train_pkls[1:]:\n    print(i)\n    train_df = train_df.append(pd.read_pickle(i))\n    train_df = train_df.astype(np.float32)\n    gc.collect()\n    \ny = train_labels.loc[train_df.index.values].values.astype(np.int8)\ntrain_df = train_df.drop(['D_64_1', 'D_66_0', 'D_68_0'], axis=1).astype(np.float32)\nprint(train_df.shape, y.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T21:44:36.279813Z","iopub.execute_input":"2022-06-17T21:44:36.280159Z","iopub.status.idle":"2022-06-17T21:45:12.570151Z","shell.execute_reply.started":"2022-06-17T21:44:36.280132Z","shell.execute_reply":"2022-06-17T21:45:12.569284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**XGBOOST**","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\ndef amex_metric(y_true, y_pred):\n    labels     = np.transpose(np.array([y_true, y_pred]))\n    labels     = labels[labels[:, 1].argsort()[::-1]]\n    weights    = np.where(labels[:,0]==0, 20, 1)\n    cut_vals   = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four   = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n\n    gini = [0,0]\n    for i in [1,0]:\n        labels         = np.transpose(np.array([y_true, y_pred]))\n        labels         = labels[labels[:, i].argsort()[::-1]]\n        weight         = np.where(labels[:,0]==0, 20, 1)\n        weight_random  = np.cumsum(weight / np.sum(weight))\n        total_pos      = np.sum(labels[:, 0] *  weight)\n        cum_pos_found  = np.cumsum(labels[:, 0] * weight)\n        lorentz        = cum_pos_found / total_pos\n        gini[i]        = np.sum((lorentz - weight_random) * weight)\n\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\n\ndef create_model(seed):\n    return xgb.XGBClassifier(\n        n_estimators = 1500,#\n        max_depth = 4,#\n        learning_rate = 0.07, #\n        subsample = 0.9, #\n        colsample_bytree = 0.3, #\n        min_child_weight = 7, #\n        reg_alpha = 2, \n        reg_lambda = 0.5, \n        tree_method ='gpu_hist',\n        predictor = 'gpu_predictor',\n        eval_metric = amex_metric,\n        random_state = SEED)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T21:45:24.309795Z","iopub.execute_input":"2022-06-17T21:45:24.310140Z","iopub.status.idle":"2022-06-17T21:45:25.084651Z","shell.execute_reply.started":"2022-06-17T21:45:24.310111Z","shell.execute_reply":"2022-06-17T21:45:25.083851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAINING**\n\nIf I try to use KFold CV the notebook craches due to shortage of system memory so I preferred to go with 75%-25% Holdout set.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(train_df, y,\n                                                    stratify=y, \n                                                    test_size=0.20)\ndel train_df, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-17T21:45:33.441936Z","iopub.execute_input":"2022-06-17T21:45:33.442321Z","iopub.status.idle":"2022-06-17T21:45:39.084861Z","shell.execute_reply.started":"2022-06-17T21:45:33.442272Z","shell.execute_reply":"2022-06-17T21:45:39.084034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(SEED)\nmodel.fit(X_train, y_train,\n         eval_set=[(X_train, y_train), (X_val, y_val)],\n         #early_stopping_rounds=200,\n         verbose=45)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T21:49:08.892254Z","iopub.execute_input":"2022-06-17T21:49:08.892991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h4>Now let's plot the feature importance</h4>","metadata":{}},{"cell_type":"code","source":"feature_important = model.get_booster().get_score(importance_type='weight')\nkeys = list(feature_important.keys())\nvalues = list(feature_important.values())\n\ndata = pd.DataFrame(data=values, index=keys, columns=[\"score\"]).sort_values(by = \"score\", ascending=False)\ndata.nlargest(25, columns=\"score\").plot(kind='barh', figsize = (20,10)) ## plot top 40 features","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:22:34.747129Z","iopub.execute_input":"2022-06-07T19:22:34.747517Z","iopub.status.idle":"2022-06-07T19:22:35.347408Z","shell.execute_reply.started":"2022-06-07T19:22:34.747481Z","shell.execute_reply":"2022-06-07T19:22:35.346552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SUBMISSION**","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\nsubmission['customer_ID_encoded'] = train_labels['customer_ID'] = submission['customer_ID'].apply(lambda x: int(x[-16:], 16)).astype(np.int64)\nsubmission.set_axis(submission['customer_ID_encoded'], inplace=True)\nsubmission = submission.drop(['customer_ID_encoded'], axis=1)\nsubmission['prediction'] = submission['prediction'].astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:22:35.351678Z","iopub.execute_input":"2022-06-07T19:22:35.352372Z","iopub.status.idle":"2022-06-07T19:22:37.622135Z","shell.execute_reply.started":"2022-06-07T19:22:35.352305Z","shell.execute_reply":"2022-06-07T19:22:37.62131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ncustomer_ids_list = []\npreds_list = []\nfor t in test_pkls:\n    test_df = pd.read_pickle(t)\n    customer_ids = test_df.axes[0].values\n    customer_ids = submission.loc[customer_ids]['customer_ID'].values\n    customer_ids_list.extend(customer_ids)\n    preds = model.predict_proba(test_df)[:, 1]\n    preds_list.extend(preds)\n    gc.collect()\n\npreds_list = np.array(preds_list).reshape(-1, 1)\ncustomer_ids_list = np.array(customer_ids_list).reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:22:37.623476Z","iopub.execute_input":"2022-06-07T19:22:37.623869Z","iopub.status.idle":"2022-06-07T19:24:22.298064Z","shell.execute_reply.started":"2022-06-07T19:22:37.623833Z","shell.execute_reply":"2022-06-07T19:24:22.297072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame(data=np.concatenate([customer_ids_list, preds_list], axis=1), columns=['customer_ID', 'prediction'])\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T19:24:22.29964Z","iopub.execute_input":"2022-06-07T19:24:22.300019Z","iopub.status.idle":"2022-06-07T19:24:27.112561Z","shell.execute_reply.started":"2022-06-07T19:24:22.299982Z","shell.execute_reply":"2022-06-07T19:24:27.11166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thanks","metadata":{}}]}