{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### In this notebook, we use [RAPIDS cudf](https://github.com/rapidsai/cudf) to create a bunch of useful features and train XGB models. The entire pipeline is lightning-fast thanks to GPU end-to-end acceleration. Train time is 20 mins and test time is 5 mins. The CV is score is `0.795` and LB score is `0.795`","metadata":{}},{"cell_type":"markdown","source":"### What you might find useful from this notebook:\n### - Super fast pipeline. LB 0.795 in 25 mins!\n### - \"After-pay\" features. It makes intuitive semse that subtracting the payments from balance/spend etc provides new information about the users' behavior.\n### - Feature selection and hyperparameter tuning. Hundreds of GPU hours are burned to get these numbers. :P\n### - Scalable streaming prediction. Each time only a chunk of test data is read, processed and predicted. If more features are added, you could simply make `chunks` bigger and never worry about GPU out of memory ","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cudf\nimport cupy\nimport xgboost as xgb\nimport numpy as np\nfrom tqdm import tqdm\ncudf.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  **Data Preprocessing**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.ensemble import RandomForestRegressor\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-07-12T06:30:21.534544Z","iopub.execute_input":"2022-07-12T06:30:21.535868Z","iopub.status.idle":"2022-07-12T06:30:22.819967Z","shell.execute_reply.started":"2022-07-12T06:30:21.535821Z","shell.execute_reply":"2022-07-12T06:30:22.819089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in ['test','train']:\n    df = pd.read_parquet(f'/kaggle/input/amex-data-integer-dtypes-parquet-format/{i}.parquet')\n    print(df.shape[0])\n    for index, row in df.iteritems():\n        missing = row.isnull().sum()\n        missing_rate = row.isnull().sum()/(row.index.stop+1)\n        if missing_rate<0.7:\n            df = df.drop(index,axis=1)   # delete missing_rate<0.7\n    sindex = np.argsort(df.isnull().sum()).values\n    # 进行缺失值的填补，利用随机森林进行填补缺失值\n    for i in sindex :\n        if df.iloc[:,i].isnull().sum() == 0 :\n            continue\n        df_00 = df\n        fillc = df_00.iloc[:, i]\n        df_00 = df_00.iloc[:,df_00.columns!=df_00.columns[i]]\n\n    #在下面的是使用了0来对特征矩阵中的缺失值的填补，\n        df_0 = SimpleImputer(missing_values=np.nan\n                            ,strategy=\"constant\"\n                            ,fill_value=0\n                            ).fit_transform(df_00)\n        Ytrain = fillc[fillc.notnull()]\n        Ytest = fillc[fillc.isnull()]\n\n        Xtrain = df_0[Ytrain.index,:]\n        Xtest = df_0[Ytest.index,:]\n\n        rfc = RandomForestRegressor()\n        rfc.fit(Xtrain, Ytrain)\n        Ypredict = rfc.predict(Xtest)\n\n        df.loc[df.iloc[:,i].isnull(),df.columns[i]] = Ypredict\n    df.to_parquet(f'{i}_fe_plus_plus.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test = pd.read_parquet('/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet')\ntrain = pd.read_parquet('/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T06:30:37.199018Z","iopub.execute_input":"2022-07-12T06:30:37.199964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sindex = np.argsort(train.isnull().sum()).values\nsindex","metadata":{"execution":{"iopub.status.busy":"2022-07-12T06:20:23.393401Z","iopub.execute_input":"2022-07-12T06:20:23.394388Z","iopub.status.idle":"2022-07-12T06:20:25.265453Z","shell.execute_reply.started":"2022-07-12T06:20:23.394333Z","shell.execute_reply":"2022-07-12T06:20:25.264597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 进行缺失值的填补，利用随机森林进行填补缺失值\nfor i in sindex :\n    if train.iloc[:,i].isnull().sum() == 0 :\n        continue\n    df = train\n    fillc = df.iloc[:, i]\n    df = df.iloc[:,df.columns!=df.columns[i]]\n\n#在下面的是使用了0来对特征矩阵中的缺失值的填补，\n    df_0 = SimpleImputer(missing_values=np.nan\n                        ,strategy=\"constant\"\n                        ,fill_value=0\n                        ).fit_transform(df)\n    Ytrain = fillc[fillc.notnull()]\n    Ytest = fillc[fillc.isnull()]\n    \n    Xtrain = df_0[Ytrain.index,:]\n    Xtest = df_0[Ytest.index,:]\n    \n    rfc = RandomForestRegressor()\n    rfc.fit(Xtrain, Ytrain)\n    Ypredict = rfc.predict(Xtest)\n    \n    train.loc[data_copy.iloc[:,i].isnull(),data_copy.columns[i]] = Ypredict","metadata":{"execution":{"iopub.status.busy":"2022-07-12T06:20:28.767189Z","iopub.execute_input":"2022-07-12T06:20:28.767572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_copy.isnull().sum()  #检查是否还有缺失值","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"def get_not_used():\n    # cid is the label encode of customer_ID\n    # row_id indicates the order of rows\n    return ['row_id', 'customer_ID', 'target', 'cid', 'S_2']\n    \ndef preprocess(df):\n    df['row_id'] = cupy.arange(df.shape[0])\n    not_used = get_not_used()\n    cat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120',\n                'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\n    for col in df.columns:\n        if col not in not_used+cat_cols:\n            df[col] = df[col].round(2)\n\n    # compute \"after pay\" features\n    for bcol in [f'B_{i}' for i in [11,14,17]]+['D_39','D_131']+[f'S_{i}' for i in [16,23]]:\n        for pcol in ['P_2','P_3']:\n            if bcol in df.columns:\n                df[f'{bcol}-{pcol}'] = df[bcol] - df[pcol]\n\n    df['S_2'] = cudf.to_datetime(df['S_2'])\n    df['cid'], _ = df.customer_ID.factorize()\n        \n    num_cols = [col for col in df.columns if col not in cat_cols+not_used]\n    \n    dgs = add_stats_step(df, num_cols)\n        \n    # cudf merge changes row orders\n    # restore the original row order by sorting row_id\n    df = df.sort_values('row_id')\n    df = df.drop(['row_id'],axis=1)\n    return df, dgs\n\ndef add_stats_step(df, cols):\n    n = 50\n    dgs = []\n    for i in range(0,len(cols),n):\n        s = i\n        e = min(s+n, len(cols))\n        dg = add_stats_one_shot(df, cols[s:e])\n        dgs.append(dg)\n    return dgs\n\ndef add_stats_one_shot(df, cols):\n    stats = ['mean','std']\n    dg = df.groupby('customer_ID').agg({col:stats for col in cols})\n    out_cols = []\n    for col in cols:\n        out_cols.extend([f'{col}_{s}' for s in stats])\n    dg.columns = out_cols\n    dg = dg.reset_index()\n    return dg\n\ndef load_test_iter(path, chunks=4):\n    \n    test_rows = 11363762\n    chunk_rows = test_rows // chunks\n    \n    test = cudf.read_parquet(f'{path}/test.parquet',\n                             columns=['customer_ID','S_2'],\n                             num_rows=test_rows)\n    test = get_segment(test)\n    start = 0\n    while start < test.shape[0]:\n        if start+chunk_rows < test.shape[0]:\n            end = test['cus_count'].values[start+chunk_rows]\n        else:\n            end = test['cus_count'].values[-1]\n        end = int(end)\n        df = cudf.read_parquet(f'{path}/test.parquet',\n                               num_rows = end-start, skiprows=start)\n        start = end\n        yield process_data(df)\n    \n\ndef load_train(path):\n    train = cudf.read_parquet(f'{path}/train.parquet')\n    \n    train = process_data(train)\n    trainl = cudf.read_csv(f'../input/amex-default-prediction/train_labels.csv')\n    train = train.merge(trainl, on='customer_ID', how='left')\n    return train\n\ndef process_data(df):\n    df,dgs = preprocess(df)\n    df = df.drop_duplicates('customer_ID',keep='last')\n    for dg in dgs:\n        df = df.merge(dg, on='customer_ID', how='left')\n    diff_cols = [col for col in df.columns if col.endswith('_diff')]\n    df = df.drop(diff_cols,axis=1)\n    return df\n\ndef get_segment(test):\n    dg = test.groupby('customer_ID').agg({'S_2':'count'})\n    dg.columns = ['cus_count']\n    dg = dg.reset_index()\n    dg['cid'],_ = dg['customer_ID'].factorize()\n    dg = dg.sort_values('cid')\n    dg['cus_count'] = dg['cus_count'].cumsum()\n    \n    test = test.merge(dg, on='customer_ID', how='left')\n    test = test.sort_values(['cid','S_2'])\n    assert test['cus_count'].values[-1] == test.shape[0]\n    return test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGB Params and utility functions","metadata":{}},{"cell_type":"code","source":"def xgb_train(x, y, xt, yt):\n    print(\"# of features:\", x.shape[1])\n    assert x.shape[1] == xt.shape[1]\n    dtrain = xgb.DMatrix(data=x, label=y)\n    dvalid = xgb.DMatrix(data=xt, label=yt)\n    params = {\n            'objective': 'binary:logistic', \n            'tree_method': 'gpu_hist', \n            'max_depth': 7,\n            'subsample':0.88,\n            'colsample_bytree': 0.5,\n            'gamma':1.5,\n            'min_child_weight':8,\n            'lambda':70,\n            'eta':0.03,\n    }\n    watchlist = [(dtrain, 'train'), (dvalid, 'eval')]\n    bst = xgb.train(params, dtrain=dtrain,\n                num_boost_round=2600,evals=watchlist,\n                early_stopping_rounds=500, feval=xgb_amex, maximize=True,\n                verbose_eval=100)\n    print('best ntree_limit:', bst.best_ntree_limit)\n    print('best score:', bst.best_score)\n    return bst.predict(dvalid, iteration_range=(0,bst.best_ntree_limit)), bst","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Metrics","metadata":{}},{"cell_type":"code","source":"def xgb_amex(y_pred, y_true):\n    return 'amex', amex_metric_np(y_pred,y_true.get_label())\n\n# Created by https://www.kaggle.com/yunchonggan\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\ndef amex_metric_np(preds: np.ndarray, target: np.ndarray) -> float:\n    indices = np.argsort(preds)[::-1]\n    preds, target = preds[indices], target[indices]\n\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_mask = cum_norm_weight <= 0.04\n    d = np.sum(target[four_pct_mask]) / np.sum(target)\n\n    weighted_target = target * weight\n    lorentz = (weighted_target / weighted_target.sum()).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    n_pos = np.sum(target)\n    n_neg = target.shape[0] - n_pos\n    gini_max = 10 * n_neg * (n_pos + 20 * n_neg - 19) / (n_pos + 20 * n_neg)\n\n    g = gini / gini_max\n    return 0.5 * (g + d)\n\n# we still need the official metric since the faster version above is slightly off\nimport pandas as pd\ndef amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df['target'] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df['target'].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df['target'] * df['weight']).sum()\n        df['cum_pos_found'] = (df['target'] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    g = normalized_weighted_gini(y_true, y_pred)\n    d = top_four_percent_captured(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data and add feature","metadata":{}},{"cell_type":"code","source":"%%time\n\npath = '../input/amex-data-integer-dtypes-parquet-format'\ntrain = load_train(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train XGB in K-folds","metadata":{}},{"cell_type":"code","source":"%%time\n\nnot_used = get_not_used()\nnot_used = [i for i in not_used if i in train.columns]\nmsgs = {}\nfolds = 4\nscore = 0\n\nfor i in range(folds):\n    mask = train['cid']%folds == i\n    tr,va = train[~mask], train[mask]\n    \n    x, y = tr.drop(not_used, axis=1), tr['target']\n    xt, yt = va.drop(not_used, axis=1), va['target']\n    yp, bst = xgb_train(x, y, xt, yt)\n    bst.save_model(f'xgb_{i}.json')\n    amex_score = amex_metric(pd.DataFrame({'target':yt.values.get()}), \n                                    pd.DataFrame({'prediction':yp}))\n    msg = f\"Fold {i} amex {amex_score:.4f}\"\n    print(msg)\n    score += amex_score\nscore /= folds\nprint(f\"Average amex score: {score:.4f}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ndel tr,va","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncids = []\nyps = []\nchunks = 4\nfor df in tqdm(load_test_iter(path,chunks),total=chunks):\n    cids.append(df['customer_ID'])\n    not_used = [i for i in not_used if i in df.columns]\n\n    yp = 0\n    for i in range(folds):\n        bst = xgb.Booster()\n        bst.load_model(f'xgb_{i}.json')\n        dx = xgb.DMatrix(df.drop(not_used, axis=1))\n        print('best ntree_limit:', bst.best_ntree_limit)\n        yp += bst.predict(dx, iteration_range=(0,bst.best_ntree_limit))\n    yps.append(yp/folds)\n    \ndf = cudf.DataFrame()\ndf['customer_ID'] = cudf.concat(cids)\ndf['prediction'] = np.concatenate(yps)\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('sub.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}