{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### In this notebook, we use [RAPIDS cudf](https://github.com/rapidsai/cudf) to create a bunch of useful features and train XGB models. The entire pipeline is lightning-fast thanks to GPU end-to-end acceleration. Train time is 20 mins and test time is 5 mins. The CV is score is `0.795` and LB score is `0.795`","metadata":{}},{"cell_type":"markdown","source":"### What you might find useful from this notebook:\n### - Super fast pipeline. LB 0.795 in 25 mins!\n### - \"After-pay\" features. It makes intuitive semse that subtracting the payments from balance/spend etc provides new information about the users' behavior.\n### - Feature selection and hyperparameter tuning. Hundreds of GPU hours are burned to get these numbers. :P\n### - Scalable streaming prediction. Each time only a chunk of test data is read, processed and predicted. If more features are added, you could simply make `chunks` bigger and never worry about GPU out of memory ","metadata":{}},{"cell_type":"code","source":"'''\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-10T07:09:24.423069Z","iopub.execute_input":"2022-07-10T07:09:24.423517Z","iopub.status.idle":"2022-07-10T07:09:24.433506Z","shell.execute_reply.started":"2022-07-10T07:09:24.423478Z","shell.execute_reply":"2022-07-10T07:09:24.432702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cudf\nimport cupy\nimport xgboost as xgb\nimport numpy as np\nfrom tqdm import tqdm\ncudf.__version__","metadata":{"execution":{"iopub.status.busy":"2022-07-10T07:00:43.318938Z","iopub.execute_input":"2022-07-10T07:00:43.319329Z","iopub.status.idle":"2022-07-10T07:00:44.753486Z","shell.execute_reply.started":"2022-07-10T07:00:43.319277Z","shell.execute_reply":"2022-07-10T07:00:44.752473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def get_not_used():\n    # cid is the label encode of customer_ID\n    # row_id indicates the order of rows\n    return ['row_id', 'customer_ID', 'target', 'cid', 'S_2']\n    \ndef preprocess(df):\n    df['row_id'] = cupy.arange(df.shape[0])\n    not_used = get_not_used()\n    cat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120',\n                'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\n    for col in df.columns:\n        if col not in not_used+cat_cols:\n            df[col] = df[col].round(2)\n\n    # compute \"after pay\" features\n    for bcol in [f'B_{i}' for i in [11,14,17]]+['D_39','D_131']+[f'S_{i}' for i in [16,23]]:\n        for pcol in ['P_2','P_3']:\n            if bcol in df.columns:\n                df[f'{bcol}-{pcol}'] = df[bcol] - df[pcol]\n\n    df['S_2'] = cudf.to_datetime(df['S_2'])\n    df['cid'], _ = df.customer_ID.factorize()\n        \n    num_cols = [col for col in df.columns if col not in cat_cols+not_used]\n    \n    dgs = add_stats_step(df, num_cols)\n        \n    # cudf merge changes row orders\n    # restore the original row order by sorting row_id\n    df = df.sort_values('row_id')\n    df = df.drop(['row_id'],axis=1)\n    return df, dgs\n\ndef add_stats_step(df, cols):\n    n = 50\n    dgs = []\n    for i in range(0,len(cols),n):\n        s = i\n        e = min(s+n, len(cols))\n        dg = add_stats_one_shot(df, cols[s:e])\n        dgs.append(dg)\n    return dgs\n\ndef add_stats_one_shot(df, cols):\n    stats = ['mean','std']\n    dg = df.groupby('customer_ID').agg({col:stats for col in cols})\n    out_cols = []\n    for col in cols:\n        out_cols.extend([f'{col}_{s}' for s in stats])\n    dg.columns = out_cols\n    dg = dg.reset_index()\n    return dg\n\ndef load_test_iter(path, chunks=4):\n    \n    test_rows = 11363762\n    chunk_rows = test_rows // chunks\n    \n    test = cudf.read_parquet(f'{path}/test.parquet',\n                             columns=['customer_ID','S_2'],\n                             num_rows=test_rows)\n    test = get_segment(test)\n    start = 0\n    while start < test.shape[0]:\n        if start+chunk_rows < test.shape[0]:\n            end = test['cus_count'].values[start+chunk_rows]\n        else:\n            end = test['cus_count'].values[-1]\n        end = int(end)\n        df = cudf.read_parquet(f'{path}/test.parquet',\n                               num_rows = end-start, skiprows=start)\n        start = end\n        yield process_data(df)\n    \n\ndef load_train(path):\n    train = cudf.read_parquet(f'{path}/train.parquet')\n    \n    train = process_data(train)\n    trainl = cudf.read_csv(f'../input/amex-default-prediction/train_labels.csv')\n    train = train.merge(trainl, on='customer_ID', how='left')\n    return train\n\ndef process_data(df):\n    df,dgs = preprocess(df)\n    df = df.drop_duplicates('customer_ID',keep='last')\n    for dg in dgs:\n        df = df.merge(dg, on='customer_ID', how='left')\n    diff_cols = [col for col in df.columns if col.endswith('_diff')]\n    df = df.drop(diff_cols,axis=1)\n    return df\n\ndef get_segment(test):\n    dg = test.groupby('customer_ID').agg({'S_2':'count'})\n    dg.columns = ['cus_count']\n    dg = dg.reset_index()\n    dg['cid'],_ = dg['customer_ID'].factorize()\n    dg = dg.sort_values('cid')\n    dg['cus_count'] = dg['cus_count'].cumsum()\n    \n    test = test.merge(dg, on='customer_ID', how='left')\n    test = test.sort_values(['cid','S_2'])\n    assert test['cus_count'].values[-1] == test.shape[0]\n    return test","metadata":{"execution":{"iopub.status.busy":"2022-07-10T07:00:46.49641Z","iopub.execute_input":"2022-07-10T07:00:46.496775Z","iopub.status.idle":"2022-07-10T07:00:46.519966Z","shell.execute_reply.started":"2022-07-10T07:00:46.496745Z","shell.execute_reply":"2022-07-10T07:00:46.518857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGB Params and utility functions","metadata":{}},{"cell_type":"markdown","source":"#### Metrics","metadata":{}},{"cell_type":"code","source":"'''\ndef xgb_train(x, y, xt, yt):\n    print(\"# of features:\", x.shape[1])\n    assert x.shape[1] == xt.shape[1]\n    dtrain = xgb.DMatrix(data=x, label=y)\n    dvalid = xgb.DMatrix(data=xt, label=yt)\n    params = {\n            'objective': 'binary:logistic', \n            'tree_method': 'gpu_hist', \n            'max_depth': 7,\n            'subsample':0.88,\n            'colsample_bytree': 0.5,\n            'gamma':1.5,\n            'min_child_weight':8,\n            'lambda':70,\n            'eta':0.03,\n    }\n    watchlist = [(dtrain, 'train'), (dvalid, 'eval')]\n    bst = xgb.train(params, dtrain=dtrain,\n                num_boost_round=2600,evals=watchlist,\n                early_stopping_rounds=500, feval=xgb_amex, maximize=True,\n                verbose_eval=100)\n    print('best ntree_limit:', bst.best_ntree_limit)\n    print('best score:', bst.best_score)\n    return bst.predict(dvalid, iteration_range=(0,bst.best_ntree_limit)), bst\n\ndef lgb_train(x, y, xt, yt):\n    print(\"# of features:\", x.shape[1])\n    assert x.shape[1] == xt.shape[1]\n    lgb_train = lgb.Dataset(x, y, categorical_feature = cat_features)\n    lgb_valid = lgb.Dataset(xt, yt, categorical_feature = cat_features)\n    params = {\n        'objective': 'binary',\n        'metric': CFG.metric,\n        'boosting': CFG.boosting_type,\n        'seed': CFG.seed,\n        'num_leaves': 100,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.20,\n        'bagging_freq': 10,\n        'bagging_fraction': 0.50,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40,\n        }\n    model = lgb.train(\n        params = params,\n        train_set = lgb_train,\n        num_boost_round = 10500,\n        valid_sets = [lgb_train, lgb_valid],\n        early_stopping_rounds = 1500,\n        verbose_eval = 500,\n        feval = lgb_amex_metric\n        )\n\n    params = {\n            'objective': 'binary:logistic', \n            'tree_method': 'gpu_hist', \n            'max_depth': 7,\n            'subsample':0.88,\n            'colsample_bytree': 0.5,\n            'gamma':1.5,\n            'min_child_weight':8,\n            'lambda':70,\n            'eta':0.03,\n    }\n    watchlist = [(dtrain, 'train'), (dvalid, 'eval')]\n    bst = xgb.train(params, dtrain=dtrain,\n                num_boost_round=2600,evals=watchlist,\n                early_stopping_rounds=500, feval=xgb_amex, maximize=True,\n                verbose_eval=100)\n    print('best ntree_limit:', bst.best_ntree_limit)\n    print('best score:', bst.best_score)\n    return bst.predict(dvalid, iteration_range=(0,bst.best_ntree_limit)), bst\n'''","metadata":{"execution":{"iopub.status.busy":"2022-07-10T06:27:39.92612Z","iopub.execute_input":"2022-07-10T06:27:39.926482Z","iopub.status.idle":"2022-07-10T06:27:39.941615Z","shell.execute_reply.started":"2022-07-10T06:27:39.926452Z","shell.execute_reply":"2022-07-10T06:27:39.940874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef xgb_amex(y_pred, y_true):\n    return 'amex', amex_metric_np(y_pred,y_true.get_label())\n\n# Created by https://www.kaggle.com/yunchonggan\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\ndef amex_metric_np(preds: np.ndarray, target: np.ndarray) -> float:\n    indices = np.argsort(preds)[::-1]\n    preds, target = preds[indices], target[indices]\n\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_mask = cum_norm_weight <= 0.04\n    d = np.sum(target[four_pct_mask]) / np.sum(target)\n\n    weighted_target = target * weight\n    lorentz = (weighted_target / weighted_target.sum()).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    n_pos = np.sum(target)\n    n_neg = target.shape[0] - n_pos\n    gini_max = 10 * n_neg * (n_pos + 20 * n_neg - 19) / (n_pos + 20 * n_neg)\n\n    g = gini / gini_max\n    return 0.5 * (g + d)\n\n# we still need the official metric since the faster version above is slightly off\nimport pandas as pd\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-07-10T06:27:46.386795Z","iopub.execute_input":"2022-07-10T06:27:46.387546Z","iopub.status.idle":"2022-07-10T06:27:46.407135Z","shell.execute_reply.started":"2022-07-10T06:27:46.387509Z","shell.execute_reply":"2022-07-10T06:27:46.406256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data and add feature","metadata":{}},{"cell_type":"code","source":"'''\n%%time\n\npath = '../input/amex-data-integer-dtypes-parquet-format'\ntrain = load_train(path)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-07-10T07:01:04.702963Z","iopub.execute_input":"2022-07-10T07:01:04.703365Z","iopub.status.idle":"2022-07-10T07:01:16.747124Z","shell.execute_reply.started":"2022-07-10T07:01:04.703322Z","shell.execute_reply":"2022-07-10T07:01:16.746252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train XGB in K-folds","metadata":{}},{"cell_type":"code","source":"'''\n%%time\n\nnot_used = get_not_used()\nnot_used = [i for i in not_used if i in train.columns]\nmsgs = {}\nfolds = 4\nscore = 0\n\nfor i in range(folds):\n    mask = train['cid']%folds == i\n    tr,va = train[~mask], train[mask]\n    \n    x, y = tr.drop(not_used, axis=1), tr['target']\n    xt, yt = va.drop(not_used, axis=1), va['target']\n    yp, bst = xgb_train(x, y, xt, yt)\n    bst.save_model(f'xgb_{i}.json')\n    amex_score = amex_metric(pd.DataFrame({'target':yt.values.get()}), \n                                    pd.DataFrame({'prediction':yp}))\n    msg = f\"Fold {i} amex {amex_score:.4f}\"\n    print(msg)\n    score += amex_score\nscore /= folds\nprint(f\"Average amex score: {score:.4f}\")\n'''","metadata":{"execution":{"iopub.status.busy":"2022-07-10T05:54:40.027253Z","iopub.execute_input":"2022-07-10T05:54:40.027916Z","iopub.status.idle":"2022-07-10T06:12:40.601479Z","shell.execute_reply.started":"2022-07-10T05:54:40.027873Z","shell.execute_reply":"2022-07-10T06:12:40.600305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del train\n# del tr,va","metadata":{"execution":{"iopub.status.busy":"2022-07-10T06:32:58.215202Z","iopub.execute_input":"2022-07-10T06:32:58.215573Z","iopub.status.idle":"2022-07-10T06:32:58.234438Z","shell.execute_reply.started":"2022-07-10T06:32:58.215542Z","shell.execute_reply":"2022-07-10T06:32:58.233205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npath = '../input/amex-data-integer-dtypes-parquet-format'\ntrain = load_train(path)\ntrain.to_parquet('train.parquet')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncids = []\nyps = []\nchunks = 4\ni = 0\nfor df in tqdm(load_test_iter(path,chunks),total=chunks):\n    cids.append(df['customer_ID'])\n#     not_used = [i for i in not_used if i in df.columns]\n\n    \n    if i == 0:\n        test_0 = df\n        test_0.to_parquet('test_0.parquet')\n    if i == 1:\n        test_1 = df\n        test_1.to_parquet('test_1.parquet')\n    if i == 2:\n        test_2 = df\n        test_2.to_parquet('test_2.parquet')\n    if i == 3:\n        test_3 = df\n        test_3.to_parquet('test_3.parquet')\n        \n    i += 1\n    \n#     yp = 0\n#     for i in range(folds):\n#         bst = xgb.Booster()\n#         bst.load_model(f'xgb_{i}.json')\n#         dx = xgb.DMatrix(df.drop(not_used, axis=1))\n#         print('best ntree_limit:', bst.best_ntree_limit)\n#         yp += bst.predict(dx, iteration_range=(0,bst.best_ntree_limit))\n#     yps.append(yp/folds)\n\n# test_all.append(test)\n\n    \n# df = cudf.DataFrame()\n# df['customer_ID'] = cudf.concat(cids)\n# df['prediction'] = np.concatenate(yps)\n# df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-10T06:54:45.644538Z","iopub.execute_input":"2022-07-10T06:54:45.644973Z","iopub.status.idle":"2022-07-10T06:56:01.896918Z","shell.execute_reply.started":"2022-07-10T06:54:45.644935Z","shell.execute_reply":"2022-07-10T06:56:01.896189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('sub.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-07T15:44:38.928833Z","iopub.execute_input":"2022-06-07T15:44:38.929465Z","iopub.status.idle":"2022-06-07T15:44:38.940583Z","shell.execute_reply.started":"2022-06-07T15:44:38.929424Z","shell.execute_reply":"2022-06-07T15:44:38.939727Z"},"trusted":true},"execution_count":null,"outputs":[]}]}