{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Lag Features Are All You Need\n\n> **Credits:** Based on [this](https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7963) amazing notebook.\n\nOK. Maybe not **all** you need.\n<br>\nBut they improve `LightGBM`!\n_____\n\n\nThis notebook stated as an ensemble of `LightGBM` + `Catboost` + `XGB` but while running it I discovered an interestin idea that worked really well.\n\n### Lag Features\n\nOn this competition we get information about clients of AMEX over time. \nMost high scoring notebooks on this competiion focused on aggregating the information per client and create a single row of extracted features: One for each client.\n\n**One of such agg function is `last`**.\n\nQuick examination revealed that the `last` feature is extreamly powerful at predicting if the client defaults or not (well.. make sense..). \nSo I took this two steps further: \n\n- **First feature:** Just like the `last` feature: I added a `first` feature. \n- **\"Lag\" fearures:** to capture the change over time about each client I calculated two features for every `first`, `last` pair:\n     - **Last - First:** The change since we first see the client to the last time we see the client.\n     - **Last / First:** The fractional difference since we first see the client to the last time we see the client.\n\nThis improved my `LightGBM` model to the point that it overtook the whole `LightGBM` + `Catboost` + `XGB` ensemble.\n\nI uploaded a dataset containing the extracted lag features and updated the final model predictions (only `LightGBM` this time) for everyone to play with. \n\n<br>\n\n_____\n\n**Next Experiement (currently running):** More \"lag features\" variations - Also take in consideration other indices of the time-series. will keep you updated.\n_____\n\n<br>\n\n\n\n","metadata":{"papermill":{"duration":0.003629,"end_time":"2022-06-19T23:15:37.769935","exception":false,"start_time":"2022-06-19T23:15:37.766306","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"papermill":{"duration":0.002235,"end_time":"2022-06-19T23:15:37.774992","exception":false,"start_time":"2022-06-19T23:15:37.772757","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nimport itertools\n\n# ====================================================\n# Read & preprocess data and save it to disk\n# ====================================================\ndef read_preprocess_data():\n    train = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')\n    features = train.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    num_features = [col for col in features if col not in cat_features]\n    \n    # Train FE\n    print('Starting train feature extraction')\n    train_num_agg = train.groupby(\"customer_ID\")[num_features].agg(['first', 'mean', 'std', 'min', 'max', 'last'])\n    train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace = True)\n\n    # Lag Features\n    for col in train_num_agg:\n        if 'last' in col and col.replace('last', 'first') in train_num_agg:\n            train_num_agg[col + '_lag_sub'] = train_num_agg[col] - train_num_agg[col.replace('last', 'first')]\n            train_num_agg[col + '_lag_div'] = train_num_agg[col] / train_num_agg[col.replace('last', 'first')]\n\n    train_cat_agg = train.groupby(\"customer_ID\")[cat_features].agg(['count', 'first', 'last', 'nunique'])\n    train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace = True)\n    \n    train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\n    train = train_num_agg.merge(train_cat_agg, how = 'inner', on = 'customer_ID').merge(train_labels, how = 'inner', on = 'customer_ID')\n    print('Train shape: ', train.shape)    \n    del train_num_agg, train_cat_agg        \n    gc.collect()\n    \n    # Test FE\n    test = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet')\n    print('Starting test feature extraction')\n    test_num_agg = test.groupby(\"customer_ID\")[num_features].agg(['first', 'mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    test_num_agg.reset_index(inplace = True)\n\n    # Lag Features\n    for col in test_num_agg:\n        if 'last' in col and col.replace('last', 'first') in test_num_agg:\n            test_num_agg[col + '_lag_sub'] = test_num_agg[col] - test_num_agg[col.replace('last', 'first')]\n            test_num_agg[col + '_lag_div'] = test_num_agg[col] / test_num_agg[col.replace('last', 'first')]\n\n    test_cat_agg = test.groupby(\"customer_ID\")[cat_features].agg(['count', 'first', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    test_cat_agg.reset_index(inplace = True)\n    \n    test = test_num_agg.merge(test_cat_agg, how = 'inner', on = 'customer_ID')\n    print('Test shape: ', test.shape)\n    del test_num_agg, test_cat_agg\n    gc.collect()\n    \n    \n    # Save files to disk\n    train.to_parquet('train_fe_plus_plus.parquet')\n    test.to_parquet('test_fe_plus_plus.parquet')\n    \n# Read & Preprocess Data\n# read_preprocess_data()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.16385,"end_time":"2022-06-19T23:15:37.941388","exception":false,"start_time":"2022-06-19T23:15:37.777538","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-03T16:36:49.935597Z","iopub.execute_input":"2022-07-03T16:36:49.936366Z","iopub.status.idle":"2022-07-03T16:36:50.113435Z","shell.execute_reply.started":"2022-07-03T16:36:49.936242Z","shell.execute_reply":"2022-07-03T16:36:50.112486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training & Inference","metadata":{"papermill":{"duration":0.002398,"end_time":"2022-06-19T23:15:37.946621","exception":false,"start_time":"2022-06-19T23:15:37.944223","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport joblib\nimport random\nimport warnings\nimport itertools\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport xgboost as xgb\nimport lightgbm as lgb\nwarnings.filterwarnings('ignore')\nfrom itertools import combinations\npd.set_option('display.width', 1000)\npd.set_option('display.max_rows', 500)\nfrom catboost import CatBoostClassifier\npd.set_option('display.max_columns', 500)\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\nclass CFG:\n    input_dir = '../input/amex-fe/'\n    seed = 42\n    n_folds = 5\n    target = 'target'\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\ndef read_data():\n    train = pd.read_parquet(CFG.input_dir + 'train_fe_plus_plus.parquet')\n    test = pd.read_parquet(CFG.input_dir + 'test_fe_plus_plus.parquet')\n    return train, test\n\ndef amex_metric(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:,0]==0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n    gini = [0,0]\n    for i in [1,0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:,0]==0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] *  weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\ndef amex_metric_np(preds, target):\n    indices = np.argsort(preds)[::-1]\n    preds, target = preds[indices], target[indices]\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_mask = cum_norm_weight <= 0.04\n    d = np.sum(target[four_pct_mask]) / np.sum(target)\n    weighted_target = target * weight\n    lorentz = (weighted_target / weighted_target.sum()).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n    n_pos = np.sum(target)\n    n_neg = target.shape[0] - n_pos\n    gini_max = 10 * n_neg * (n_pos + 20 * n_neg - 19) / (n_pos + 20 * n_neg)\n    g = gini / gini_max\n    return 0.5 * (g + d)","metadata":{"papermill":{"duration":1.91354,"end_time":"2022-06-19T23:15:39.862701","exception":false,"start_time":"2022-06-19T23:15:37.949161","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-03T16:36:50.115074Z","iopub.execute_input":"2022-07-03T16:36:50.115416Z","iopub.status.idle":"2022-07-03T16:36:52.698333Z","shell.execute_reply.started":"2022-07-03T16:36:50.115388Z","shell.execute_reply":"2022-07-03T16:36:52.697548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training LightGBM (DART) Model\n\n- Final predictions output uploaded as a public dataset. ","metadata":{}},{"cell_type":"code","source":"def lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return 'amex_metric', amex_metric(y_true, y_pred), True\n\ndef train_and_evaluate(train, test):\n    # Label encode categorical features\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\"\n    ]\n    cat_features = [f\"{cf}_last\" for cf in cat_features]\n    for cat_col in cat_features:\n        encoder = LabelEncoder()\n        train[cat_col] = encoder.fit_transform(train[cat_col])\n        test[cat_col] = encoder.transform(test[cat_col])\n    # Round last float features to 2 decimal place\n    num_cols = list(train.dtypes[(train.dtypes == 'float32') | (train.dtypes == 'float64')].index)\n    num_cols = [col for col in num_cols if 'last' in col]\n    for col in num_cols:\n        train[col + '_round2'] = train[col].round(2)\n        test[col + '_round2'] = test[col].round(2)\n    # Get feature list\n    features = [col for col in train.columns if col not in ['customer_ID', CFG.target]]\n    params = {\n        'objective': 'binary',\n        'metric': \"binary_logloss\",\n        'boosting': 'dart',\n        'seed': CFG.seed,\n        'num_leaves': 100,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.20,\n        'bagging_freq': 10,\n        'bagging_fraction': 0.50,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40\n        }\n    # Create a numpy array to store test predictions\n    test_predictions = np.zeros(len(test))\n    # Create a numpy array to store out of folds predictions\n    oof_predictions = np.zeros(len(train))\n    kfold = StratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.seed)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):\n        print(' ')\n        print('-'*50)\n        print(f'Training fold {fold} with {len(features)} features...')\n        x_train, x_val = train[features].iloc[trn_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[CFG.target].iloc[trn_ind], train[CFG.target].iloc[val_ind]\n        lgb_train = lgb.Dataset(x_train, y_train, categorical_feature = cat_features)\n        lgb_valid = lgb.Dataset(x_val, y_val, categorical_feature = cat_features)\n        model = lgb.train(\n            params = params,\n            train_set = lgb_train,\n            num_boost_round = 10500,\n            valid_sets = [lgb_train, lgb_valid],\n            early_stopping_rounds = 100,\n            verbose_eval = 500,\n            feval = lgb_amex_metric\n            )\n        # Save best model\n        joblib.dump(model, f'lgbm_fold{fold}_seed{CFG.seed}.pkl')\n        # Predict validation\n        val_pred = model.predict(x_val)\n        # Add to out of folds array\n        oof_predictions[val_ind] = val_pred\n        # Predict the test set\n        test_pred = model.predict(test[features])\n        test_predictions += test_pred / CFG.n_folds\n        # Compute fold metric\n        score = amex_metric(y_val, val_pred)\n        print(f'Our fold {fold} CV score is {score}')\n        del x_train, x_val, y_train, y_val, lgb_train, lgb_valid\n        gc.collect()\n    # Compute out of folds metric\n    score = amex_metric(train[CFG.target], oof_predictions)\n    print(f'Our out of folds CV score is {score}')\n    # Create a dataframe to store out of folds predictions\n    oof_df = pd.DataFrame({'customer_ID': train['customer_ID'], 'target': train[CFG.target], 'prediction': oof_predictions})\n    oof_df.to_csv(f'oof_lgbm_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)\n    # Create a dataframe to store test prediction\n    test_df = pd.DataFrame({'customer_ID': test['customer_ID'], 'prediction': test_predictions})\n    test_df.to_csv(f'test_lgbm_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)\n\n# seed_everything(CFG.seed)\n# train, test = read_data()\n# train_and_evaluate(train, test)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-03T16:36:52.700595Z","iopub.execute_input":"2022-07-03T16:36:52.701018Z","iopub.status.idle":"2022-07-03T16:36:52.731615Z","shell.execute_reply.started":"2022-07-03T16:36:52.700987Z","shell.execute_reply":"2022-07-03T16:36:52.730798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction\n\n- Replace / comment-out this to use your own predictions from the model in the above cell.","metadata":{"papermill":{"duration":0.00263,"end_time":"2022-06-19T23:15:39.870669","exception":false,"start_time":"2022-06-19T23:15:39.868039","status":"completed"},"tags":[],"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"import os\nimport pandas as pd\n\ndf_1 = pd.read_csv('../input/amex-predictions/test_lgbm_baseline_5fold_seed42.csv')\ndf_1.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":7.116467,"end_time":"2022-06-19T23:15:46.98989","exception":false,"start_time":"2022-06-19T23:15:39.873423","status":"completed"},"tags":[],"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-03T16:36:52.796711Z","iopub.execute_input":"2022-07-03T16:36:52.797315Z","iopub.status.idle":"2022-07-03T16:37:04.221485Z","shell.execute_reply.started":"2022-07-03T16:36:52.797274Z","shell.execute_reply":"2022-07-03T16:37:04.220504Z"},"trusted":true},"execution_count":null,"outputs":[]}]}