{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### kaggle kernels pull jsaguiar/lightgbm-7th-place-solution","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## KAGGLE HOME CREDIT DEFAULT RISK COMPETITION\n\nAdapted from one of the models used in 7th place solutiuon ensemble.\n\nFor more details about our solution please check this [discussion](https://www.kaggle.com/c/home-credit-default-risk/discussion/64580)\n\nAnother similar version is also available at [GitHub](https://github.com/js-aguiar/home-credit-default-competition)\n\nThis model uses LightGBM with goss and label encode for the application's categorical features. Other tables are using one-hot encode with mean, sum and a few different functions to aggregate. The main idea as to add more time related features like last application and last X months aggregations. There are also aggreagtions for specific loan types and status as well as ratios between tables.","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport time\nimport numpy as np\nimport pandas as pd\nfrom contextlib import contextmanager\nimport multiprocessing as mp\nfrom functools import partial\nfrom scipy.stats import kurtosis, iqr, skew\nfrom lightgbm import LGBMClassifier\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:39.962826Z","iopub.execute_input":"2022-08-04T06:42:39.963309Z","iopub.status.idle":"2022-08-04T06:42:41.592569Z","shell.execute_reply.started":"2022-08-04T06:42:39.963215Z","shell.execute_reply":"2022-08-04T06:42:41.591329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main(debug= False):\n    num_rows = 30000 if debug else None\n    with timer(\"application_train and application_test\"):\n        df = get_train_test(DATA_DIRECTORY, num_rows= num_rows)\n        print(\"Application dataframe shape: \", df.shape)\n    \n    with timer(\"Bureau and bureau_balance data\"):\n        bureau_df = get_bureau(DATA_DIRECTORY, num_rows= num_rows)\n        df = pd.merge(df, bureau_df, on='SK_ID_CURR', how='left')\n        print(\"Bureau dataframe shape: \", bureau_df.shape)\n        del bureau_df; gc.collect()\n    \n    with timer(\"previous_application\"):\n        prev_df = get_previous_applications(DATA_DIRECTORY, num_rows)\n        df = pd.merge(df, prev_df, on='SK_ID_CURR', how='left')\n        print(\"Previous dataframe shape: \", prev_df.shape)\n        del prev_df; gc.collect()\n    \n    with timer(\"previous applications balances\"):\n        pos = get_pos_cash(DATA_DIRECTORY, num_rows)\n        df = pd.merge(df, pos, on='SK_ID_CURR', how='left')\n        print(\"Pos-cash dataframe shape: \", pos.shape)\n        del pos; gc.collect()\n        ins = get_installment_payments(DATA_DIRECTORY, num_rows)\n        df = pd.merge(df, ins, on='SK_ID_CURR', how='left')\n        print(\"Installments dataframe shape: \", ins.shape)\n        del ins; gc.collect()\n        cc = get_credit_card(DATA_DIRECTORY, num_rows)\n        df = pd.merge(df, cc, on='SK_ID_CURR', how='left')\n        print(\"Credit card dataframe shape: \", cc.shape)\n        del cc; gc.collect()\n        \n    # Add ratios and groupby bwtween different tables\n    df = add_ratios_features(df)\n    df = reduce_memory(df)\n    lgbm_categorical_feat = [\n        'CODE_GENDER', 'FLAG_OWN_CAR', 'NAME_CONTRACT_TYPE', 'NAME_EDUCATION_TYPE',\n        'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'NAME_INCOME_TYPE', 'OCCUPATION_TYPE',\n        'ORGANIZATION_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'NAME_TYPE_SUITE', 'WALLSMATERIAL_MODE']\n    \n    with timer(\"Run LightGBM\"):\n        feat_importance = kfold_lightgbm_sklearn(df, lgbm_categorical_feat)\n        print(feat_importance)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:41.595173Z","iopub.execute_input":"2022-08-04T06:42:41.595672Z","iopub.status.idle":"2022-08-04T06:42:41.611759Z","shell.execute_reply.started":"2022-08-04T06:42:41.595627Z","shell.execute_reply":"2022-08-04T06:42:41.610681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_ratios_features(df):\n    # CREDIT TO INCOME RATIO\n    df['BUREAU_INCOME_CREDIT_RATIO'] = df['BUREAU_AMT_CREDIT_SUM_MEAN'] / df['AMT_INCOME_TOTAL']\n    df['BUREAU_ACTIVE_CREDIT_TO_INCOME_RATIO'] = df['BUREAU_ACTIVE_AMT_CREDIT_SUM_SUM'] / df['AMT_INCOME_TOTAL']\n    \n    # PREVIOUS TO CURRENT CREDIT RATIO\n    df['CURRENT_TO_APPROVED_CREDIT_MIN_RATIO'] = df['APPROVED_AMT_CREDIT_MIN'] / df['AMT_CREDIT']\n    df['CURRENT_TO_APPROVED_CREDIT_MAX_RATIO'] = df['APPROVED_AMT_CREDIT_MAX'] / df['AMT_CREDIT']\n    df['CURRENT_TO_APPROVED_CREDIT_MEAN_RATIO'] = df['APPROVED_AMT_CREDIT_MEAN'] / df['AMT_CREDIT']\n    \n    # PREVIOUS TO CURRENT ANNUITY RATIO\n    df['CURRENT_TO_APPROVED_ANNUITY_MAX_RATIO'] = df['APPROVED_AMT_ANNUITY_MAX'] / df['AMT_ANNUITY']\n    df['CURRENT_TO_APPROVED_ANNUITY_MEAN_RATIO'] = df['APPROVED_AMT_ANNUITY_MEAN'] / df['AMT_ANNUITY']\n    df['PAYMENT_MIN_TO_ANNUITY_RATIO'] = df['INS_AMT_PAYMENT_MIN'] / df['AMT_ANNUITY']\n    df['PAYMENT_MAX_TO_ANNUITY_RATIO'] = df['INS_AMT_PAYMENT_MAX'] / df['AMT_ANNUITY']\n    df['PAYMENT_MEAN_TO_ANNUITY_RATIO'] = df['INS_AMT_PAYMENT_MEAN'] / df['AMT_ANNUITY']\n    \n    # PREVIOUS TO CURRENT CREDIT TO ANNUITY RATIO\n    df['CTA_CREDIT_TO_ANNUITY_MAX_RATIO'] = df['APPROVED_CREDIT_TO_ANNUITY_RATIO_MAX'] / df[\n        'CREDIT_TO_ANNUITY_RATIO']\n    df['CTA_CREDIT_TO_ANNUITY_MEAN_RATIO'] = df['APPROVED_CREDIT_TO_ANNUITY_RATIO_MEAN'] / df[\n        'CREDIT_TO_ANNUITY_RATIO']\n    \n    # DAYS DIFFERENCES AND RATIOS\n    df['DAYS_DECISION_MEAN_TO_BIRTH'] = df['APPROVED_DAYS_DECISION_MEAN'] / df['DAYS_BIRTH']\n    df['DAYS_CREDIT_MEAN_TO_BIRTH'] = df['BUREAU_DAYS_CREDIT_MEAN'] / df['DAYS_BIRTH']\n    df['DAYS_DECISION_MEAN_TO_EMPLOYED'] = df['APPROVED_DAYS_DECISION_MEAN'] / df['DAYS_EMPLOYED']\n    df['DAYS_CREDIT_MEAN_TO_EMPLOYED'] = df['BUREAU_DAYS_CREDIT_MEAN'] / df['DAYS_EMPLOYED']\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:41.613052Z","iopub.execute_input":"2022-08-04T06:42:41.613974Z","iopub.status.idle":"2022-08-04T06:42:41.627268Z","shell.execute_reply.started":"2022-08-04T06:42:41.613942Z","shell.execute_reply":"2022-08-04T06:42:41.626437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LIGHTGBM MODEL","metadata":{}},{"cell_type":"code","source":"def kfold_lightgbm_sklearn(data, categorical_feature = None):\n    df = data[data['TARGET'].notnull()]\n    test = data[data['TARGET'].isnull()]\n    df = df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))\n    test = test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))\n    print(\"Train/valid shape: {}, test shape: {}\".format(df.shape, test.shape))\n    del_features = ['TARGET', 'SK_ID_CURR', 'SK_ID_BUREAU', 'SK_ID_PREV', 'index', 'level_0']\n    predictors = list(filter(lambda v: v not in del_features, df.columns))\n\n    if not STRATIFIED_KFOLD:\n        folds = KFold(n_splits= NUM_FOLDS, shuffle=True, random_state= RANDOM_SEED)\n    else:\n        folds = StratifiedKFold(n_splits= NUM_FOLDS, shuffle=True, random_state= RANDOM_SEED)\n\n    # Hold oof predictions, test predictions, feature importance and training/valid auc\n    oof_preds = np.zeros(df.shape[0])\n    sub_preds = np.zeros(test.shape[0])\n    importance_df = pd.DataFrame()\n    eval_results = dict()\n\n    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(df[predictors], df['TARGET'])):\n        train_x, train_y = df[predictors].iloc[train_idx], df['TARGET'].iloc[train_idx]\n        valid_x, valid_y = df[predictors].iloc[valid_idx], df['TARGET'].iloc[valid_idx]\n\n        params = {'random_state': RANDOM_SEED, 'nthread': NUM_THREADS}\n        clf = LGBMClassifier(**{**params, **LIGHTGBM_PARAMS})\n        if not categorical_feature:\n            clf.fit(train_x, train_y, eval_set=[(train_x, train_y), (valid_x, valid_y)],\n                    eval_metric='auc', verbose=400, early_stopping_rounds= EARLY_STOPPING)\n        else:\n            clf.fit(train_x, train_y, eval_set=[(train_x, train_y), (valid_x, valid_y)],\n                    eval_metric='auc', verbose=400, early_stopping_rounds=EARLY_STOPPING,\n                    feature_name= list(df[predictors].columns), categorical_feature= categorical_feature)\n\n        oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]\n        sub_preds += clf.predict_proba(test[predictors], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n\n        # Feature importance by GAIN and SPLIT\n        fold_importance = pd.DataFrame()\n        fold_importance[\"feature\"] = predictors\n        fold_importance[\"gain\"] = clf.booster_.feature_importance(importance_type='gain')\n        fold_importance[\"split\"] = clf.booster_.feature_importance(importance_type='split')\n        importance_df = pd.concat([importance_df, fold_importance], axis=0)\n        eval_results['train_{}'.format(n_fold+1)]  = clf.evals_result_['training']['auc']\n        eval_results['valid_{}'.format(n_fold+1)] = clf.evals_result_['valid_1']['auc']\n\n        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))\n        del clf, train_x, train_y, valid_x, valid_y\n        gc.collect()\n\n    print('Full AUC score %.6f' % roc_auc_score(df['TARGET'], oof_preds))\n    test['TARGET'] = sub_preds.copy()\n\n    # Get the average feature importance between folds\n    mean_importance = importance_df.groupby('feature').mean().reset_index()\n    mean_importance.sort_values(by= 'gain', ascending=False, inplace=True)\n    # Save feature importance, test predictions and oof predictions as csv\n    if GENERATE_SUBMISSION_FILES:\n\n        # Generate oof csv\n        oof = pd.DataFrame()\n        oof['SK_ID_CURR'] = df['SK_ID_CURR'].copy()\n        df['PREDICTIONS'] = oof_preds.copy()\n        df['TARGET'] = df['TARGET'].copy()\n        df.to_csv('oof{}.csv'.format(SUBMISSION_SUFIX), index=False)\n        # Save submission (test data) and feature importance\n        test[['SK_ID_CURR', 'TARGET']].to_csv('submission{}.csv'.format(SUBMISSION_SUFIX), index=False)\n        mean_importance.to_csv('feature_importance{}.csv'.format(SUBMISSION_SUFIX), index=False)\n    return mean_importance","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:41.672836Z","iopub.execute_input":"2022-08-04T06:42:41.673273Z","iopub.status.idle":"2022-08-04T06:42:41.694446Z","shell.execute_reply.started":"2022-08-04T06:42:41.673235Z","shell.execute_reply":"2022-08-04T06:42:41.693254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# APPLICATION PIPELINE","metadata":{}},{"cell_type":"code","source":"def get_train_test(path, num_rows = None):\n    \"\"\" Process application_train.csv and application_test.csv and return a pandas dataframe. \"\"\"\n    train = pd.read_csv(os.path.join(path, 'application_train.csv'), nrows= num_rows)\n    test = pd.read_csv(os.path.join(path, 'application_test.csv'), nrows= num_rows)\n    df = train.append(test)\n    del train, test; gc.collect()\n    \n    # Data cleaning\n    df = df[df['CODE_GENDER'] != 'XNA']  # 4 people with XNA code gender\n    df = df[df['AMT_INCOME_TOTAL'] < 20000000]  # Max income in test is 4M; train has a 117M value\n    df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace=True)\n    df['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)\n\n    # Flag_document features - count and kurtosis\n    docs = [f for f in df.columns if 'FLAG_DOC' in f]\n    df['DOCUMENT_COUNT'] = df[docs].sum(axis=1)\n    df['NEW_DOC_KURT'] = df[docs].kurtosis(axis=1)\n    \n    # Categorical age - based on target=1 plot\n    df['AGE_RANGE'] = df['DAYS_BIRTH'].apply(lambda x: get_age_label(x))\n\n    # New features based on External sources\n    df['EXT_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']\n    df['EXT_SOURCES_WEIGHTED'] = df.EXT_SOURCE_1 * 2 + df.EXT_SOURCE_2 * 1 + df.EXT_SOURCE_3 * 3\n    np.warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered')\n    for function_name in ['min', 'max', 'mean', 'nanmedian', 'var']:\n        feature_name = 'EXT_SOURCES_{}'.format(function_name.upper())\n        df[feature_name] = eval('np.{}'.format(function_name))(\n            df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']], axis=1)\n\n    # Credit ratios\n    df['CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']\n    df['CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']\n    \n    # Income ratios\n    df['ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']\n    df['CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']\n    df['INCOME_TO_EMPLOYED_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_EMPLOYED']\n    df['INCOME_TO_BIRTH_RATIO'] = df['AMT_INCOME_TOTAL'] / df['DAYS_BIRTH']\n   \n    # Time ratios\n    df['EMPLOYED_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']\n    df['ID_TO_BIRTH_RATIO'] = df['DAYS_ID_PUBLISH'] / df['DAYS_BIRTH']\n    df['CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']\n    df['CAR_TO_EMPLOYED_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']\n    df['PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']\n\n    # Groupby: Statistics for applications in the same group\n    group = ['ORGANIZATION_TYPE', 'NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE', 'AGE_RANGE', 'CODE_GENDER']\n    df = do_median(df, group, 'EXT_SOURCES_MEAN', 'GROUP_EXT_SOURCES_MEDIAN')\n    df = do_std(df, group, 'EXT_SOURCES_MEAN', 'GROUP_EXT_SOURCES_STD')\n    df = do_mean(df, group, 'AMT_INCOME_TOTAL', 'GROUP_INCOME_MEAN')\n    df = do_std(df, group, 'AMT_INCOME_TOTAL', 'GROUP_INCOME_STD')\n    df = do_mean(df, group, 'CREDIT_TO_ANNUITY_RATIO', 'GROUP_CREDIT_TO_ANNUITY_MEAN')\n    df = do_std(df, group, 'CREDIT_TO_ANNUITY_RATIO', 'GROUP_CREDIT_TO_ANNUITY_STD')\n    df = do_mean(df, group, 'AMT_CREDIT', 'GROUP_CREDIT_MEAN')\n    df = do_mean(df, group, 'AMT_ANNUITY', 'GROUP_ANNUITY_MEAN')\n    df = do_std(df, group, 'AMT_ANNUITY', 'GROUP_ANNUITY_STD')\n\n    # Encode categorical features (LabelEncoder)\n    df, le_encoded_cols = label_encoder(df, None)\n    df = drop_application_columns(df)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:42.533814Z","iopub.execute_input":"2022-08-04T06:42:42.534987Z","iopub.status.idle":"2022-08-04T06:42:42.552851Z","shell.execute_reply.started":"2022-08-04T06:42:42.534946Z","shell.execute_reply":"2022-08-04T06:42:42.551856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_application_columns(df):\n    \"\"\" Drop features based on permutation feature importance. \"\"\"\n    drop_list = [\n        'CNT_CHILDREN', 'CNT_FAM_MEMBERS', 'HOUR_APPR_PROCESS_START',\n        'FLAG_EMP_PHONE', 'FLAG_MOBIL', 'FLAG_CONT_MOBILE', 'FLAG_EMAIL', 'FLAG_PHONE',\n        'FLAG_OWN_REALTY', 'REG_REGION_NOT_LIVE_REGION', 'REG_REGION_NOT_WORK_REGION',\n        'REG_CITY_NOT_WORK_CITY', 'OBS_30_CNT_SOCIAL_CIRCLE', 'OBS_60_CNT_SOCIAL_CIRCLE',\n        'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_YEAR', \n        'COMMONAREA_MODE', 'NONLIVINGAREA_MODE', 'ELEVATORS_MODE', 'NONLIVINGAREA_AVG',\n        'FLOORSMIN_MEDI', 'LANDAREA_MODE', 'NONLIVINGAREA_MEDI', 'LIVINGAPARTMENTS_MODE',\n        'FLOORSMIN_AVG', 'LANDAREA_AVG', 'FLOORSMIN_MODE', 'LANDAREA_MEDI',\n        'COMMONAREA_MEDI', 'YEARS_BUILD_AVG', 'COMMONAREA_AVG', 'BASEMENTAREA_AVG',\n        'BASEMENTAREA_MODE', 'NONLIVINGAPARTMENTS_MEDI', 'BASEMENTAREA_MEDI', \n        'LIVINGAPARTMENTS_AVG', 'ELEVATORS_AVG', 'YEARS_BUILD_MEDI', 'ENTRANCES_MODE',\n        'NONLIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE', 'LIVINGAPARTMENTS_MEDI',\n        'YEARS_BUILD_MODE', 'YEARS_BEGINEXPLUATATION_AVG', 'ELEVATORS_MEDI', 'LIVINGAREA_MEDI',\n        'YEARS_BEGINEXPLUATATION_MODE', 'NONLIVINGAPARTMENTS_AVG', 'HOUSETYPE_MODE',\n        'FONDKAPREMONT_MODE', 'EMERGENCYSTATE_MODE'\n    ]\n    \n    # Drop most flag document columns\n    for doc_num in [2,4,5,6,7,9,10,11,12,13,14,15,16,17,19,20,21]:\n        drop_list.append('FLAG_DOCUMENT_{}'.format(doc_num))\n    df.drop(drop_list, axis=1, inplace=True)\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:43.016986Z","iopub.execute_input":"2022-08-04T06:42:43.018087Z","iopub.status.idle":"2022-08-04T06:42:43.026656Z","shell.execute_reply.started":"2022-08-04T06:42:43.018042Z","shell.execute_reply":"2022-08-04T06:42:43.025530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_age_label(days_birth):\n    \"\"\" Return the age group label (int). \"\"\"\n    age_years = -days_birth / 365\n    if age_years < 27: return 1\n    elif age_years < 40: return 2\n    elif age_years < 50: return 3\n    elif age_years < 65: return 4\n    elif age_years < 99: return 5\n    else: return 0","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:43.554231Z","iopub.execute_input":"2022-08-04T06:42:43.554657Z","iopub.status.idle":"2022-08-04T06:42:43.560344Z","shell.execute_reply.started":"2022-08-04T06:42:43.554622Z","shell.execute_reply":"2022-08-04T06:42:43.559514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BUREAU PIPELINE","metadata":{}},{"cell_type":"code","source":"def get_bureau(path, num_rows= None):\n    \"\"\" Process bureau.csv and bureau_balance.csv and return a pandas dataframe. \"\"\"\n    bureau = pd.read_csv(os.path.join(path, 'bureau.csv'), nrows= num_rows)\n    \n    # Credit duration and credit/account end date difference\n    bureau['CREDIT_DURATION'] = -bureau['DAYS_CREDIT'] + bureau['DAYS_CREDIT_ENDDATE']\n    bureau['ENDDATE_DIF'] = bureau['DAYS_CREDIT_ENDDATE'] - bureau['DAYS_ENDDATE_FACT']\n    \n    # Credit to debt ratio and difference\n    bureau['DEBT_PERCENTAGE'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_CREDIT_SUM_DEBT']\n    bureau['DEBT_CREDIT_DIFF'] = bureau['AMT_CREDIT_SUM'] - bureau['AMT_CREDIT_SUM_DEBT']\n    bureau['CREDIT_TO_ANNUITY_RATIO'] = bureau['AMT_CREDIT_SUM'] / bureau['AMT_ANNUITY']\n\n    # One-hot encoder\n    bureau, categorical_cols = one_hot_encoder(bureau, nan_as_category= False)\n    \n    # Join bureau balance features\n    bureau = bureau.merge(get_bureau_balance(path, num_rows), how='left', on='SK_ID_BUREAU')\n    \n    # Flag months with late payments (days past due)\n    bureau['STATUS_12345'] = 0\n    for i in range(1,6):\n        bureau['STATUS_12345'] += bureau['STATUS_{}'.format(i)]\n\n    # Aggregate by number of months in balance and merge with bureau (loan length agg)\n    features = ['AMT_CREDIT_MAX_OVERDUE', 'AMT_CREDIT_SUM_OVERDUE', 'AMT_CREDIT_SUM',\n        'AMT_CREDIT_SUM_DEBT', 'DEBT_PERCENTAGE', 'DEBT_CREDIT_DIFF', 'STATUS_0', 'STATUS_12345']\n    agg_length = bureau.groupby('MONTHS_BALANCE_SIZE')[features].mean().reset_index()\n    agg_length.rename({feat: 'LL_' + feat for feat in features}, axis=1, inplace=True)\n    bureau = bureau.merge(agg_length, how='left', on='MONTHS_BALANCE_SIZE')\n    del agg_length; gc.collect()\n\n    # General loans aggregations\n    agg_bureau = group(bureau, 'BUREAU_', BUREAU_AGG)\n    \n    # Active and closed loans aggregations\n    active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]\n    agg_bureau = group_and_merge(active,agg_bureau,'BUREAU_ACTIVE_',BUREAU_ACTIVE_AGG)\n    closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]\n    agg_bureau = group_and_merge(closed,agg_bureau,'BUREAU_CLOSED_',BUREAU_CLOSED_AGG)\n    del active, closed; gc.collect()\n    \n    # Aggregations for the main loan types\n    for credit_type in ['Consumer credit', 'Credit card', 'Mortgage', 'Car loan', 'Microloan']:\n        type_df = bureau[bureau['CREDIT_TYPE_' + credit_type] == 1]\n        prefix = 'BUREAU_' + credit_type.split(' ')[0].upper() + '_'\n        agg_bureau = group_and_merge(type_df, agg_bureau, prefix, BUREAU_LOAN_TYPE_AGG)\n        del type_df; gc.collect()\n    \n    # Time based aggregations: last x months\n    for time_frame in [6, 12]:\n        prefix = \"BUREAU_LAST{}M_\".format(time_frame)\n        time_frame_df = bureau[bureau['DAYS_CREDIT'] >= -30*time_frame]\n        agg_bureau = group_and_merge(time_frame_df, agg_bureau, prefix, BUREAU_TIME_AGG)\n        del time_frame_df; gc.collect()\n\n    # Last loan max overdue\n    sort_bureau = bureau.sort_values(by=['DAYS_CREDIT'])\n    gr = sort_bureau.groupby('SK_ID_CURR')['AMT_CREDIT_MAX_OVERDUE'].last().reset_index()\n    gr.rename({'AMT_CREDIT_MAX_OVERDUE': 'BUREAU_LAST_LOAN_MAX_OVERDUE'}, inplace=True)\n    agg_bureau = agg_bureau.merge(gr, on='SK_ID_CURR', how='left')\n    \n    # Ratios: total debt/total credit and active loans debt/ active loans credit\n    agg_bureau['BUREAU_DEBT_OVER_CREDIT'] = \\\n        agg_bureau['BUREAU_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_AMT_CREDIT_SUM_SUM']\n    agg_bureau['BUREAU_ACTIVE_DEBT_OVER_CREDIT'] = \\\n        agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_DEBT_SUM']/agg_bureau['BUREAU_ACTIVE_AMT_CREDIT_SUM_SUM']\n    return agg_bureau","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:43.875883Z","iopub.execute_input":"2022-08-04T06:42:43.876315Z","iopub.status.idle":"2022-08-04T06:42:43.895293Z","shell.execute_reply.started":"2022-08-04T06:42:43.876275Z","shell.execute_reply":"2022-08-04T06:42:43.894343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_bureau_balance(path, num_rows= None):\n    bb = pd.read_csv(os.path.join(path, 'bureau_balance.csv'), nrows= num_rows)\n    bb, categorical_cols = one_hot_encoder(bb, nan_as_category= False)\n    \n    # Calculate rate for each category with decay\n    bb_processed = bb.groupby('SK_ID_BUREAU')[categorical_cols].mean().reset_index()\n    \n    # Min, Max, Count and mean duration of payments (months)\n    agg = {'MONTHS_BALANCE': ['min', 'max', 'mean', 'size']}\n    bb_processed = group_and_merge(bb, bb_processed, '', agg, 'SK_ID_BUREAU')\n    del bb; gc.collect()\n    return bb_processed","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:44.137881Z","iopub.execute_input":"2022-08-04T06:42:44.138722Z","iopub.status.idle":"2022-08-04T06:42:44.146678Z","shell.execute_reply.started":"2022-08-04T06:42:44.138678Z","shell.execute_reply":"2022-08-04T06:42:44.145393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PREVIOUS PIPELINE","metadata":{}},{"cell_type":"code","source":"def get_previous_applications(path, num_rows= None):\n    \"\"\" Process previous_application.csv and return a pandas dataframe. \"\"\"\n    prev = pd.read_csv(os.path.join(path, 'previous_application.csv'), nrows= num_rows)\n    pay = pd.read_csv(os.path.join(path, 'installments_payments.csv'), nrows= num_rows)\n\n    # One-hot encode most important categorical features\n    ohe_columns = [\n        'NAME_CONTRACT_STATUS', 'NAME_CONTRACT_TYPE', 'CHANNEL_TYPE',\n        'NAME_TYPE_SUITE', 'NAME_YIELD_GROUP', 'PRODUCT_COMBINATION',\n        'NAME_PRODUCT_TYPE', 'NAME_CLIENT_TYPE']\n    prev, categorical_cols = one_hot_encoder(prev, ohe_columns, nan_as_category= False)\n\n    # Feature engineering: ratios and difference\n    prev['APPLICATION_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']\n    prev['APPLICATION_CREDIT_RATIO'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']\n    prev['CREDIT_TO_ANNUITY_RATIO'] = prev['AMT_CREDIT']/prev['AMT_ANNUITY']\n    prev['DOWN_PAYMENT_TO_CREDIT'] = prev['AMT_DOWN_PAYMENT'] / prev['AMT_CREDIT']\n    \n    # Interest ratio on previous application (simplified)\n    total_payment = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']\n    prev['SIMPLE_INTERESTS'] = (total_payment/prev['AMT_CREDIT'] - 1)/prev['CNT_PAYMENT']\n\n    # Active loans - approved and not complete yet (last_due 365243)\n    approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]\n    active_df = approved[approved['DAYS_LAST_DUE'] == 365243]\n    \n    # Find how much was already payed in active loans (using installments csv)\n    active_pay = pay[pay['SK_ID_PREV'].isin(active_df['SK_ID_PREV'])]\n    active_pay_agg = active_pay.groupby('SK_ID_PREV')[['AMT_INSTALMENT', 'AMT_PAYMENT']].sum()\n    active_pay_agg.reset_index(inplace= True)\n    \n    # Active loans: difference of what was payed and installments\n    active_pay_agg['INSTALMENT_PAYMENT_DIFF'] = active_pay_agg['AMT_INSTALMENT'] - active_pay_agg['AMT_PAYMENT']\n    \n    # Merge with active_df\n    active_df = active_df.merge(active_pay_agg, on= 'SK_ID_PREV', how= 'left')\n    active_df['REMAINING_DEBT'] = active_df['AMT_CREDIT'] - active_df['AMT_PAYMENT']\n    active_df['REPAYMENT_RATIO'] = active_df['AMT_PAYMENT'] / active_df['AMT_CREDIT']\n    \n    # Perform aggregations for active applications\n    active_agg_df = group(active_df, 'PREV_ACTIVE_', PREVIOUS_ACTIVE_AGG)\n    active_agg_df['TOTAL_REPAYMENT_RATIO'] = active_agg_df['PREV_ACTIVE_AMT_PAYMENT_SUM']/\\\n                                             active_agg_df['PREV_ACTIVE_AMT_CREDIT_SUM']\n    del active_pay, active_pay_agg, active_df; gc.collect()\n\n    # Change 365.243 values to nan (missing)\n    prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)\n    prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)\n    prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)\n    prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)\n    prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)\n    \n    # Days last due difference (scheduled x done)\n    prev['DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']\n    approved['DAYS_LAST_DUE_DIFF'] = approved['DAYS_LAST_DUE_1ST_VERSION'] - approved['DAYS_LAST_DUE']\n\n    # Categorical features\n    categorical_agg = {key: ['mean'] for key in categorical_cols}\n    \n    # Perform general aggregations\n    agg_prev = group(prev, 'PREV_', {**PREVIOUS_AGG, **categorical_agg})\n    \n    # Merge active loans dataframe on agg_prev\n    agg_prev = agg_prev.merge(active_agg_df, how='left', on='SK_ID_CURR')\n    del active_agg_df; gc.collect()\n    \n    # Aggregations for approved and refused loans\n    agg_prev = group_and_merge(approved, agg_prev, 'APPROVED_', PREVIOUS_APPROVED_AGG)\n    refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]\n    agg_prev = group_and_merge(refused, agg_prev, 'REFUSED_', PREVIOUS_REFUSED_AGG)\n    del approved, refused; gc.collect()\n    \n    # Aggregations for Consumer loans and Cash loans\n    for loan_type in ['Consumer loans', 'Cash loans']:\n        type_df = prev[prev['NAME_CONTRACT_TYPE_{}'.format(loan_type)] == 1]\n        prefix = 'PREV_' + loan_type.split(\" \")[0] + '_'\n        agg_prev = group_and_merge(type_df, agg_prev, prefix, PREVIOUS_LOAN_TYPE_AGG)\n        del type_df; gc.collect()\n\n    # Get the SK_ID_PREV for loans with late payments (days past due)\n    pay['LATE_PAYMENT'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT']\n    pay['LATE_PAYMENT'] = pay['LATE_PAYMENT'].apply(lambda x: 1 if x > 0 else 0)\n    dpd_id = pay[pay['LATE_PAYMENT'] > 0]['SK_ID_PREV'].unique()\n    \n    # Aggregations for loans with late payments\n    agg_dpd = group_and_merge(prev[prev['SK_ID_PREV'].isin(dpd_id)], agg_prev,\n                                    'PREV_LATE_', PREVIOUS_LATE_PAYMENTS_AGG)\n    del agg_dpd, dpd_id; gc.collect()\n    \n    # Aggregations for loans in the last x months\n    for time_frame in [12, 24]:\n        time_frame_df = prev[prev['DAYS_DECISION'] >= -30*time_frame]\n        prefix = 'PREV_LAST{}M_'.format(time_frame)\n        agg_prev = group_and_merge(time_frame_df, agg_prev, prefix, PREVIOUS_TIME_AGG)\n        del time_frame_df; gc.collect()\n    del prev; gc.collect()\n    return agg_prev","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:44.648968Z","iopub.execute_input":"2022-08-04T06:42:44.649403Z","iopub.status.idle":"2022-08-04T06:42:44.673498Z","shell.execute_reply.started":"2022-08-04T06:42:44.649365Z","shell.execute_reply":"2022-08-04T06:42:44.672576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# POS-CASH PIPELINE","metadata":{}},{"cell_type":"code","source":"def get_pos_cash(path, num_rows= None):\n    \"\"\" Process POS_CASH_balance.csv and return a pandas dataframe. \"\"\"\n    pos = pd.read_csv(os.path.join(path, 'POS_CASH_balance.csv'), nrows= num_rows)\n    pos, categorical_cols = one_hot_encoder(pos, nan_as_category= False)\n    \n    # Flag months with late payment\n    pos['LATE_PAYMENT'] = pos['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)\n    \n    # Aggregate by SK_ID_CURR\n    categorical_agg = {key: ['mean'] for key in categorical_cols}\n    pos_agg = group(pos, 'POS_', {**POS_CASH_AGG, **categorical_agg})\n    \n    # Sort and group by SK_ID_PREV\n    sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE'])\n    gp = sort_pos.groupby('SK_ID_PREV')\n    df = pd.DataFrame()\n    df['SK_ID_CURR'] = gp['SK_ID_CURR'].first()\n    df['MONTHS_BALANCE_MAX'] = gp['MONTHS_BALANCE'].max()\n    \n    # Percentage of previous loans completed and completed before initial term\n    df['POS_LOAN_COMPLETED_MEAN'] = gp['NAME_CONTRACT_STATUS_Completed'].mean()\n    df['POS_COMPLETED_BEFORE_MEAN'] = gp['CNT_INSTALMENT'].first() - gp['CNT_INSTALMENT'].last()\n    df['POS_COMPLETED_BEFORE_MEAN'] = df.apply(lambda x: 1 if x['POS_COMPLETED_BEFORE_MEAN'] > 0\n                                                and x['POS_LOAN_COMPLETED_MEAN'] > 0 else 0, axis=1)\n    \n    # Number of remaining installments (future installments) and percentage from total\n    df['POS_REMAINING_INSTALMENTS'] = gp['CNT_INSTALMENT_FUTURE'].last()\n    df['POS_REMAINING_INSTALMENTS_RATIO'] = gp['CNT_INSTALMENT_FUTURE'].last()/gp['CNT_INSTALMENT'].last()\n    \n    # Group by SK_ID_CURR and merge\n    df_gp = df.groupby('SK_ID_CURR').sum().reset_index()\n    df_gp.drop(['MONTHS_BALANCE_MAX'], axis=1, inplace= True)\n    pos_agg = pd.merge(pos_agg, df_gp, on= 'SK_ID_CURR', how= 'left')\n    del df, gp, df_gp, sort_pos; gc.collect()\n\n    # Percentage of late payments for the 3 most recent applications\n    pos = do_sum(pos, ['SK_ID_PREV'], 'LATE_PAYMENT', 'LATE_PAYMENT_SUM')\n    \n    # Last month of each application\n    last_month_df = pos.groupby('SK_ID_PREV')['MONTHS_BALANCE'].idxmax()\n    \n    # Most recent applications (last 3)\n    sort_pos = pos.sort_values(by=['SK_ID_PREV', 'MONTHS_BALANCE'])\n    gp = sort_pos.iloc[last_month_df].groupby('SK_ID_CURR').tail(3)\n    gp_mean = gp.groupby('SK_ID_CURR').mean().reset_index()\n    pos_agg = pd.merge(pos_agg, gp_mean[['SK_ID_CURR','LATE_PAYMENT_SUM']], on='SK_ID_CURR', how='left')\n\n    # Drop some useless categorical features\n    drop_features = [\n        'POS_NAME_CONTRACT_STATUS_Canceled_MEAN', 'POS_NAME_CONTRACT_STATUS_Amortized debt_MEAN',\n        'POS_NAME_CONTRACT_STATUS_XNA_MEAN']\n    pos_agg.drop(drop_features, axis=1, inplace=True)\n    return pos_agg","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:45.024967Z","iopub.execute_input":"2022-08-04T06:42:45.026045Z","iopub.status.idle":"2022-08-04T06:42:45.041498Z","shell.execute_reply.started":"2022-08-04T06:42:45.026006Z","shell.execute_reply":"2022-08-04T06:42:45.040178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# INSTALLMENTS PIPELINE","metadata":{}},{"cell_type":"code","source":"def get_installment_payments(path, num_rows= None):\n    \"\"\" Process installments_payments.csv and return a pandas dataframe. \"\"\"\n    pay = pd.read_csv(os.path.join(path, 'installments_payments.csv'), nrows= num_rows)\n    \n    # Group payments and get Payment difference\n    pay = do_sum(pay, ['SK_ID_PREV', 'NUM_INSTALMENT_NUMBER'], 'AMT_PAYMENT', 'AMT_PAYMENT_GROUPED')\n    pay['PAYMENT_DIFFERENCE'] = pay['AMT_INSTALMENT'] - pay['AMT_PAYMENT_GROUPED']\n    pay['PAYMENT_RATIO'] = pay['AMT_INSTALMENT'] / pay['AMT_PAYMENT_GROUPED']\n    pay['PAID_OVER_AMOUNT'] = pay['AMT_PAYMENT'] - pay['AMT_INSTALMENT']\n    pay['PAID_OVER'] = (pay['PAID_OVER_AMOUNT'] > 0).astype(int)\n    \n    # Payment Entry: Days past due and Days before due\n    pay['DPD'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT']\n    pay['DPD'] = pay['DPD'].apply(lambda x: 0 if x <= 0 else x)\n    pay['DBD'] = pay['DAYS_INSTALMENT'] - pay['DAYS_ENTRY_PAYMENT']\n    pay['DBD'] = pay['DBD'].apply(lambda x: 0 if x <= 0 else x)\n    \n    # Flag late payment\n    pay['LATE_PAYMENT'] = pay['DBD'].apply(lambda x: 1 if x > 0 else 0)\n    \n    # Percentage of payments that were late\n    pay['INSTALMENT_PAYMENT_RATIO'] = pay['AMT_PAYMENT'] / pay['AMT_INSTALMENT']\n    pay['LATE_PAYMENT_RATIO'] = pay.apply(lambda x: x['INSTALMENT_PAYMENT_RATIO'] if x['LATE_PAYMENT'] == 1 else 0, axis=1)\n    \n    # Flag late payments that have a significant amount\n    pay['SIGNIFICANT_LATE_PAYMENT'] = pay['LATE_PAYMENT_RATIO'].apply(lambda x: 1 if x > 0.05 else 0)\n    \n    # Flag k threshold late payments\n    pay['DPD_7'] = pay['DPD'].apply(lambda x: 1 if x >= 7 else 0)\n    pay['DPD_15'] = pay['DPD'].apply(lambda x: 1 if x >= 15 else 0)\n    \n    # Aggregations by SK_ID_CURR\n    pay_agg = group(pay, 'INS_', INSTALLMENTS_AGG)\n\n    # Installments in the last x months\n    for months in [36, 60]:\n        recent_prev_id = pay[pay['DAYS_INSTALMENT'] >= -30*months]['SK_ID_PREV'].unique()\n        pay_recent = pay[pay['SK_ID_PREV'].isin(recent_prev_id)]\n        prefix = 'INS_{}M_'.format(months)\n        pay_agg = group_and_merge(pay_recent, pay_agg, prefix, INSTALLMENTS_TIME_AGG)\n\n    # Last x periods trend features\n    group_features = ['SK_ID_CURR', 'SK_ID_PREV', 'DPD', 'LATE_PAYMENT',\n                      'PAID_OVER_AMOUNT', 'PAID_OVER', 'DAYS_INSTALMENT']\n    gp = pay[group_features].groupby('SK_ID_CURR')\n    func = partial(trend_in_last_k_instalment_features, periods= INSTALLMENTS_LAST_K_TREND_PERIODS)\n    g = parallel_apply(gp, func, index_name='SK_ID_CURR', chunk_size=10000).reset_index()\n    pay_agg = pay_agg.merge(g, on='SK_ID_CURR', how='left')\n\n    # Last loan features\n    g = parallel_apply(gp, installments_last_loan_features, index_name='SK_ID_CURR', chunk_size=10000).reset_index()\n    pay_agg = pay_agg.merge(g, on='SK_ID_CURR', how='left')\n    return pay_agg","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:45.468298Z","iopub.execute_input":"2022-08-04T06:42:45.468730Z","iopub.status.idle":"2022-08-04T06:42:45.485080Z","shell.execute_reply.started":"2022-08-04T06:42:45.468696Z","shell.execute_reply":"2022-08-04T06:42:45.483815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def trend_in_last_k_instalment_features(gr, periods):\n    gr_ = gr.copy()\n    gr_.sort_values(['DAYS_INSTALMENT'], ascending=False, inplace=True)\n    features = {}\n\n    for period in periods:\n        gr_period = gr_.iloc[:period]\n        features = add_trend_feature(features, gr_period, 'DPD',\n                                           '{}_TREND_'.format(period))\n        features = add_trend_feature(features, gr_period, 'PAID_OVER_AMOUNT',\n                                           '{}_TREND_'.format(period))\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:45.823056Z","iopub.execute_input":"2022-08-04T06:42:45.824241Z","iopub.status.idle":"2022-08-04T06:42:45.830940Z","shell.execute_reply.started":"2022-08-04T06:42:45.824182Z","shell.execute_reply":"2022-08-04T06:42:45.829850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def installments_last_loan_features(gr):\n    gr_ = gr.copy()\n    gr_.sort_values(['DAYS_INSTALMENT'], ascending=False, inplace=True)\n    last_installment_id = gr_['SK_ID_PREV'].iloc[0]\n    gr_ = gr_[gr_['SK_ID_PREV'] == last_installment_id]\n\n    features = {}\n    features = add_features_in_group(features, gr_, 'DPD',\n                                     ['sum', 'mean', 'max', 'std'],\n                                     'LAST_LOAN_')\n    features = add_features_in_group(features, gr_, 'LATE_PAYMENT',\n                                     ['count', 'mean'],\n                                     'LAST_LOAN_')\n    features = add_features_in_group(features, gr_, 'PAID_OVER_AMOUNT',\n                                     ['sum', 'mean', 'max', 'min', 'std'],\n                                     'LAST_LOAN_')\n    features = add_features_in_group(features, gr_, 'PAID_OVER',\n                                     ['count', 'mean'],\n                                     'LAST_LOAN_')\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:46.117728Z","iopub.execute_input":"2022-08-04T06:42:46.118125Z","iopub.status.idle":"2022-08-04T06:42:46.126331Z","shell.execute_reply.started":"2022-08-04T06:42:46.118091Z","shell.execute_reply":"2022-08-04T06:42:46.125283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CREDIT CARD PIPELINE","metadata":{}},{"cell_type":"code","source":"def get_credit_card(path, num_rows= None):\n    \"\"\" Process credit_card_balance.csv and return a pandas dataframe. \"\"\"\n    cc = pd.read_csv(os.path.join(path, 'credit_card_balance.csv'), nrows= num_rows)\n    cc, cat_cols = one_hot_encoder(cc, nan_as_category=False)\n    cc.rename(columns={'AMT_RECIVABLE': 'AMT_RECEIVABLE'}, inplace=True)\n    \n    # Amount used from limit\n    cc['LIMIT_USE'] = cc['AMT_BALANCE'] / cc['AMT_CREDIT_LIMIT_ACTUAL']\n    \n    # Current payment / Min payment\n    cc['PAYMENT_DIV_MIN'] = cc['AMT_PAYMENT_CURRENT'] / cc['AMT_INST_MIN_REGULARITY']\n    \n    # Late payment\n    cc['LATE_PAYMENT'] = cc['SK_DPD'].apply(lambda x: 1 if x > 0 else 0)\n    \n    # How much drawing of limit\n    cc['DRAWING_LIMIT_RATIO'] = cc['AMT_DRAWINGS_ATM_CURRENT'] / cc['AMT_CREDIT_LIMIT_ACTUAL']\n    \n    # Aggregations by SK_ID_CURR\n    cc_agg = cc.groupby('SK_ID_CURR').agg(CREDIT_CARD_AGG)\n    cc_agg.columns = pd.Index(['CC_' + e[0] + \"_\" + e[1].upper() for e in cc_agg.columns.tolist()])\n    cc_agg.reset_index(inplace= True)\n\n    # Last month balance of each credit card application\n    last_ids = cc.groupby('SK_ID_PREV')['MONTHS_BALANCE'].idxmax()\n    last_months_df = cc[cc.index.isin(last_ids)]\n    cc_agg = group_and_merge(last_months_df,cc_agg,'CC_LAST_', {'AMT_BALANCE': ['mean', 'max']})\n\n    # Aggregations for last x months\n    for months in [12, 24, 48]:\n        cc_prev_id = cc[cc['MONTHS_BALANCE'] >= -months]['SK_ID_PREV'].unique()\n        cc_recent = cc[cc['SK_ID_PREV'].isin(cc_prev_id)]\n        prefix = 'INS_{}M_'.format(months)\n        cc_agg = group_and_merge(cc_recent, cc_agg, prefix, CREDIT_CARD_TIME_AGG)\n    return cc_agg","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:46.615253Z","iopub.execute_input":"2022-08-04T06:42:46.616433Z","iopub.status.idle":"2022-08-04T06:42:46.629156Z","shell.execute_reply.started":"2022-08-04T06:42:46.616376Z","shell.execute_reply":"2022-08-04T06:42:46.627891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# UTILITY FUNCTIONS","metadata":{}},{"cell_type":"code","source":"@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    print(\"{} - done in {:.0f}s\".format(name, time.time() - t0))","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:47.105399Z","iopub.execute_input":"2022-08-04T06:42:47.106246Z","iopub.status.idle":"2022-08-04T06:42:47.112343Z","shell.execute_reply.started":"2022-08-04T06:42:47.106207Z","shell.execute_reply":"2022-08-04T06:42:47.111068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def group(df_to_agg, prefix, aggregations, aggregate_by= 'SK_ID_CURR'):\n    agg_df = df_to_agg.groupby(aggregate_by).agg(aggregations)\n    agg_df.columns = pd.Index(['{}{}_{}'.format(prefix, e[0], e[1].upper())\n                               for e in agg_df.columns.tolist()])\n    return agg_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:47.457656Z","iopub.execute_input":"2022-08-04T06:42:47.458080Z","iopub.status.idle":"2022-08-04T06:42:47.464513Z","shell.execute_reply.started":"2022-08-04T06:42:47.458048Z","shell.execute_reply":"2022-08-04T06:42:47.463181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def group_and_merge(df_to_agg, df_to_merge, prefix, aggregations, aggregate_by= 'SK_ID_CURR'):\n    agg_df = group(df_to_agg, prefix, aggregations, aggregate_by= aggregate_by)\n    return df_to_merge.merge(agg_df, how='left', on= aggregate_by)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:47.892877Z","iopub.execute_input":"2022-08-04T06:42:47.894048Z","iopub.status.idle":"2022-08-04T06:42:47.900695Z","shell.execute_reply.started":"2022-08-04T06:42:47.893990Z","shell.execute_reply":"2022-08-04T06:42:47.899755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_mean(df, group_cols, counted, agg_name):\n    gp = df[group_cols + [counted]].groupby(group_cols)[counted].mean().reset_index().rename(\n        columns={counted: agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:48.351838Z","iopub.execute_input":"2022-08-04T06:42:48.352248Z","iopub.status.idle":"2022-08-04T06:42:48.359230Z","shell.execute_reply.started":"2022-08-04T06:42:48.352217Z","shell.execute_reply":"2022-08-04T06:42:48.358015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_median(df, group_cols, counted, agg_name):\n    gp = df[group_cols + [counted]].groupby(group_cols)[counted].median().reset_index().rename(\n        columns={counted: agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:48.762608Z","iopub.execute_input":"2022-08-04T06:42:48.763236Z","iopub.status.idle":"2022-08-04T06:42:48.769662Z","shell.execute_reply.started":"2022-08-04T06:42:48.763202Z","shell.execute_reply":"2022-08-04T06:42:48.768371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_std(df, group_cols, counted, agg_name):\n    gp = df[group_cols + [counted]].groupby(group_cols)[counted].std().reset_index().rename(\n        columns={counted: agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:49.047821Z","iopub.execute_input":"2022-08-04T06:42:49.048434Z","iopub.status.idle":"2022-08-04T06:42:49.054769Z","shell.execute_reply.started":"2022-08-04T06:42:49.048401Z","shell.execute_reply":"2022-08-04T06:42:49.053404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def do_sum(df, group_cols, counted, agg_name):\n    gp = df[group_cols + [counted]].groupby(group_cols)[counted].sum().reset_index().rename(\n        columns={counted: agg_name})\n    df = df.merge(gp, on=group_cols, how='left')\n    del gp\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:49.337984Z","iopub.execute_input":"2022-08-04T06:42:49.338841Z","iopub.status.idle":"2022-08-04T06:42:49.346359Z","shell.execute_reply.started":"2022-08-04T06:42:49.338790Z","shell.execute_reply":"2022-08-04T06:42:49.345344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_hot_encoder(df, categorical_columns=None, nan_as_category=True):\n    \"\"\"Create a new column for each categorical value in categorical columns. \"\"\"\n    original_columns = list(df.columns)\n    if not categorical_columns:\n        categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    df = pd.get_dummies(df, columns=categorical_columns, dummy_na=nan_as_category)\n    categorical_columns = [c for c in df.columns if c not in original_columns]\n    return df, categorical_columns","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:49.767581Z","iopub.execute_input":"2022-08-04T06:42:49.768013Z","iopub.status.idle":"2022-08-04T06:42:49.775612Z","shell.execute_reply.started":"2022-08-04T06:42:49.767977Z","shell.execute_reply":"2022-08-04T06:42:49.774074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_encoder(df, categorical_columns=None):\n    \"\"\"Encode categorical values as integers (0,1,2,3...) with pandas.factorize. \"\"\"\n    if not categorical_columns:\n        categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    for col in categorical_columns:\n        df[col], uniques = pd.factorize(df[col])\n    return df, categorical_columns","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:50.053309Z","iopub.execute_input":"2022-08-04T06:42:50.053753Z","iopub.status.idle":"2022-08-04T06:42:50.060618Z","shell.execute_reply.started":"2022-08-04T06:42:50.053713Z","shell.execute_reply":"2022-08-04T06:42:50.059280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(feature_name, aggs, features, feature_names, groupby):\n    feature_names.extend(['{}_{}'.format(feature_name, agg) for agg in aggs])\n\n    for agg in aggs:\n        if agg == 'kurt':\n            agg_func = kurtosis\n        elif agg == 'iqr':\n            agg_func = iqr\n        else:\n            agg_func = agg\n\n        g = groupby[feature_name].agg(agg_func).reset_index().rename(index=str,\n                                                                     columns={feature_name: '{}_{}'.format(feature_name,agg)})\n        features = features.merge(g, on='SK_ID_CURR', how='left')\n    return features, feature_names","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:50.362674Z","iopub.execute_input":"2022-08-04T06:42:50.363100Z","iopub.status.idle":"2022-08-04T06:42:50.370507Z","shell.execute_reply.started":"2022-08-04T06:42:50.363063Z","shell.execute_reply":"2022-08-04T06:42:50.369525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features_in_group(features, gr_, feature_name, aggs, prefix):\n    for agg in aggs:\n        if agg == 'sum':\n            features['{}{}_sum'.format(prefix, feature_name)] = gr_[feature_name].sum()\n        elif agg == 'mean':\n            features['{}{}_mean'.format(prefix, feature_name)] = gr_[feature_name].mean()\n        elif agg == 'max':\n            features['{}{}_max'.format(prefix, feature_name)] = gr_[feature_name].max()\n        elif agg == 'min':\n            features['{}{}_min'.format(prefix, feature_name)] = gr_[feature_name].min()\n        elif agg == 'std':\n            features['{}{}_std'.format(prefix, feature_name)] = gr_[feature_name].std()\n        elif agg == 'count':\n            features['{}{}_count'.format(prefix, feature_name)] = gr_[feature_name].count()\n        elif agg == 'skew':\n            features['{}{}_skew'.format(prefix, feature_name)] = skew(gr_[feature_name])\n        elif agg == 'kurt':\n            features['{}{}_kurt'.format(prefix, feature_name)] = kurtosis(gr_[feature_name])\n        elif agg == 'iqr':\n            features['{}{}_iqr'.format(prefix, feature_name)] = iqr(gr_[feature_name])\n        elif agg == 'median':\n            features['{}{}_median'.format(prefix, feature_name)] = gr_[feature_name].median()\n    return features\n","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:50.712389Z","iopub.execute_input":"2022-08-04T06:42:50.713060Z","iopub.status.idle":"2022-08-04T06:42:50.725828Z","shell.execute_reply.started":"2022-08-04T06:42:50.713011Z","shell.execute_reply":"2022-08-04T06:42:50.724129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_trend_feature(features, gr, feature_name, prefix):\n    y = gr[feature_name].values\n    try:\n        x = np.arange(0, len(y)).reshape(-1, 1)\n        lr = LinearRegression()\n        lr.fit(x, y)\n        trend = lr.coef_[0]\n    except:\n        trend = np.nan\n    features['{}{}'.format(prefix, feature_name)] = trend\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:51.350503Z","iopub.execute_input":"2022-08-04T06:42:51.351722Z","iopub.status.idle":"2022-08-04T06:42:51.358090Z","shell.execute_reply.started":"2022-08-04T06:42:51.351679Z","shell.execute_reply":"2022-08-04T06:42:51.357053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parallel_apply(groups, func, index_name='Index', num_workers=0, chunk_size=100000):\n    if num_workers <= 0: num_workers = NUM_THREADS\n    #n_chunks = np.ceil(1.0 * groups.ngroups / chunk_size)\n    indeces, features = [], []\n    for index_chunk, groups_chunk in chunk_groups(groups, chunk_size):\n        with mp.pool.Pool(num_workers) as executor:\n            features_chunk = executor.map(func, groups_chunk)\n        features.extend(features_chunk)\n        indeces.extend(index_chunk)\n\n    features = pd.DataFrame(features)\n    features.index = indeces\n    features.index.name = index_name\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:51.796882Z","iopub.execute_input":"2022-08-04T06:42:51.797279Z","iopub.status.idle":"2022-08-04T06:42:51.804740Z","shell.execute_reply.started":"2022-08-04T06:42:51.797247Z","shell.execute_reply":"2022-08-04T06:42:51.803857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunk_groups(groupby_object, chunk_size):\n    n_groups = groupby_object.ngroups\n    group_chunk, index_chunk = [], []\n    for i, (index, df) in enumerate(groupby_object):\n        group_chunk.append(df)\n        index_chunk.append(index)\n        if (i + 1) % chunk_size == 0 or i + 1 == n_groups:\n            group_chunk_, index_chunk_ = group_chunk.copy(), index_chunk.copy()\n            group_chunk, index_chunk = [], []\n            yield index_chunk_, group_chunk_","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:52.081403Z","iopub.execute_input":"2022-08-04T06:42:52.082306Z","iopub.status.idle":"2022-08-04T06:42:52.089484Z","shell.execute_reply.started":"2022-08-04T06:42:52.082265Z","shell.execute_reply":"2022-08-04T06:42:52.088172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_memory(df):\n    \"\"\"Reduce memory usage of a dataframe by setting data types. \"\"\"\n    start_mem = df.memory_usage().sum() / 1024 ** 2\n    print('Initial df memory usage is {:.2f} MB for {} columns'\n          .format(start_mem, len(df.columns)))\n\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type != object:\n            cmin = df[col].min()\n            cmax = df[col].max()\n            if str(col_type)[:3] == 'int':\n                # Can use unsigned int here too\n                if cmin > np.iinfo(np.int8).min and cmax < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif cmin > np.iinfo(np.int16).min and cmax < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif cmin > np.iinfo(np.int32).min and cmax < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif cmin > np.iinfo(np.int64).min and cmax < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if cmin > np.finfo(np.float16).min and cmax < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif cmin > np.finfo(np.float32).min and cmax < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    memory_reduction = 100 * (start_mem - end_mem) / start_mem\n    print('Final memory usage is: {:.2f} MB - decreased by {:.1f}%'.format(end_mem, memory_reduction))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:52.398278Z","iopub.execute_input":"2022-08-04T06:42:52.399274Z","iopub.status.idle":"2022-08-04T06:42:52.411945Z","shell.execute_reply.started":"2022-08-04T06:42:52.399235Z","shell.execute_reply":"2022-08-04T06:42:52.411032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CONFIGUARTIONS","metadata":{}},{"cell_type":"code","source":"# GENERAL CONFIGURATIONS\nNUM_THREADS = 4\nDATA_DIRECTORY = \"../input/home-credit-default-risk/\"\nSUBMISSION_SUFIX = \"_model2_04\"\n\n# INSTALLMENTS TREND PERIODS\nINSTALLMENTS_LAST_K_TREND_PERIODS = [12, 24, 60, 120]\n\n# LIGHTGBM CONFIGURATION AND HYPER-PARAMETERS\nGENERATE_SUBMISSION_FILES = True\nSTRATIFIED_KFOLD = False\nRANDOM_SEED = 737851\nNUM_FOLDS = 10\nEARLY_STOPPING = 100\n\nLIGHTGBM_PARAMS = {\n    'boosting_type': 'goss',\n    'n_estimators': 10000,\n    'learning_rate': 0.005134,\n    'num_leaves': 54,\n    'max_depth': 10,\n    'subsample_for_bin': 240000,\n    'reg_alpha': 0.436193,\n    'reg_lambda': 0.479169,\n    'colsample_bytree': 0.508716,\n    'min_split_gain': 0.024766,\n    'subsample': 1,\n    'is_unbalance': False,\n    'silent':-1,\n    'verbose':-1\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:52.696257Z","iopub.execute_input":"2022-08-04T06:42:52.697571Z","iopub.status.idle":"2022-08-04T06:42:52.705203Z","shell.execute_reply.started":"2022-08-04T06:42:52.697518Z","shell.execute_reply":"2022-08-04T06:42:52.704387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# AGGREGATIONS\n\nBUREAU_AGG = {\n    'SK_ID_BUREAU': ['nunique'],\n    'DAYS_CREDIT': ['min', 'max', 'mean'],\n    'DAYS_CREDIT_ENDDATE': ['min', 'max'],\n    'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],\n    'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],\n    'AMT_CREDIT_SUM_DEBT': ['max', 'mean', 'sum'],\n    'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean', 'sum'],\n    'AMT_ANNUITY': ['mean'],\n    'DEBT_CREDIT_DIFF': ['mean', 'sum'],\n    'MONTHS_BALANCE_MEAN': ['mean', 'var'],\n    'MONTHS_BALANCE_SIZE': ['mean', 'sum'],\n    # Categorical\n    'STATUS_0': ['mean'],\n    'STATUS_1': ['mean'],\n    'STATUS_12345': ['mean'],\n    'STATUS_C': ['mean'],\n    'STATUS_X': ['mean'],\n    'CREDIT_ACTIVE_Active': ['mean'],\n    'CREDIT_ACTIVE_Closed': ['mean'],\n    'CREDIT_ACTIVE_Sold': ['mean'],\n    'CREDIT_TYPE_Consumer credit': ['mean'],\n    'CREDIT_TYPE_Credit card': ['mean'],\n    'CREDIT_TYPE_Car loan': ['mean'],\n    'CREDIT_TYPE_Mortgage': ['mean'],\n    'CREDIT_TYPE_Microloan': ['mean'],\n    # Group by loan duration features (months)\n    'LL_AMT_CREDIT_SUM_OVERDUE': ['mean'],\n    'LL_DEBT_CREDIT_DIFF': ['mean'],\n    'LL_STATUS_12345': ['mean'],\n}\n\nBUREAU_ACTIVE_AGG = {\n    'DAYS_CREDIT': ['max', 'mean'],\n    'DAYS_CREDIT_ENDDATE': ['min', 'max'],\n    'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],\n    'AMT_CREDIT_SUM': ['max', 'sum'],\n    'AMT_CREDIT_SUM_DEBT': ['mean', 'sum'],\n    'AMT_CREDIT_SUM_OVERDUE': ['max', 'mean'],\n    'DAYS_CREDIT_UPDATE': ['min', 'mean'],\n    'DEBT_PERCENTAGE': ['mean'],\n    'DEBT_CREDIT_DIFF': ['mean'],\n    'CREDIT_TO_ANNUITY_RATIO': ['mean'],\n    'MONTHS_BALANCE_MEAN': ['mean', 'var'],\n    'MONTHS_BALANCE_SIZE': ['mean', 'sum'],\n}\n\nBUREAU_CLOSED_AGG = {\n    'DAYS_CREDIT': ['max', 'var'],\n    'DAYS_CREDIT_ENDDATE': ['max'],\n    'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],\n    'AMT_CREDIT_SUM_OVERDUE': ['mean'],\n    'AMT_CREDIT_SUM': ['max', 'mean', 'sum'],\n    'AMT_CREDIT_SUM_DEBT': ['max', 'sum'],\n    'DAYS_CREDIT_UPDATE': ['max'],\n    'ENDDATE_DIF': ['mean'],\n    'STATUS_12345': ['mean'],\n}\n\nBUREAU_LOAN_TYPE_AGG = {\n    'DAYS_CREDIT': ['mean', 'max'],\n    'AMT_CREDIT_MAX_OVERDUE': ['mean', 'max'],\n    'AMT_CREDIT_SUM': ['mean', 'max'],\n    'AMT_CREDIT_SUM_DEBT': ['mean', 'max'],\n    'DEBT_PERCENTAGE': ['mean'],\n    'DEBT_CREDIT_DIFF': ['mean'],\n    'DAYS_CREDIT_ENDDATE': ['max'],\n}\n\nBUREAU_TIME_AGG = {\n    'AMT_CREDIT_MAX_OVERDUE': ['max', 'mean'],\n    'AMT_CREDIT_SUM_OVERDUE': ['mean'],\n    'AMT_CREDIT_SUM': ['max', 'sum'],\n    'AMT_CREDIT_SUM_DEBT': ['mean', 'sum'],\n    'DEBT_PERCENTAGE': ['mean'],\n    'DEBT_CREDIT_DIFF': ['mean'],\n    'STATUS_0': ['mean'],\n    'STATUS_12345': ['mean'],\n}\n\nPREVIOUS_AGG = {\n    'SK_ID_PREV': ['nunique'],\n    'AMT_ANNUITY': ['min', 'max', 'mean'],\n    'AMT_DOWN_PAYMENT': ['max', 'mean'],\n    'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],\n    'RATE_DOWN_PAYMENT': ['max', 'mean'],\n    'DAYS_DECISION': ['min', 'max', 'mean'],\n    'CNT_PAYMENT': ['max', 'mean'],\n    'DAYS_TERMINATION': ['max'],\n    # Engineered features\n    'CREDIT_TO_ANNUITY_RATIO': ['mean', 'max'],\n    'APPLICATION_CREDIT_DIFF': ['min', 'max', 'mean'],\n    'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean', 'var'],\n    'DOWN_PAYMENT_TO_CREDIT': ['mean'],\n}\n\nPREVIOUS_ACTIVE_AGG = {\n    'SK_ID_PREV': ['nunique'],\n    'SIMPLE_INTERESTS': ['mean'],\n    'AMT_ANNUITY': ['max', 'sum'],\n    'AMT_APPLICATION': ['max', 'mean'],\n    'AMT_CREDIT': ['sum'],\n    'AMT_DOWN_PAYMENT': ['max', 'mean'],\n    'DAYS_DECISION': ['min', 'mean'],\n    'CNT_PAYMENT': ['mean', 'sum'],\n    'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],\n    # Engineered features\n    'AMT_PAYMENT': ['sum'],\n    'INSTALMENT_PAYMENT_DIFF': ['mean', 'max'],\n    'REMAINING_DEBT': ['max', 'mean', 'sum'],\n    'REPAYMENT_RATIO': ['mean'],\n}\n\nPREVIOUS_APPROVED_AGG = {\n    'SK_ID_PREV': ['nunique'],\n    'AMT_ANNUITY': ['min', 'max', 'mean'],\n    'AMT_CREDIT': ['min', 'max', 'mean'],\n    'AMT_DOWN_PAYMENT': ['max'],\n    'AMT_GOODS_PRICE': ['max'],\n    'HOUR_APPR_PROCESS_START': ['min', 'max'],\n    'DAYS_DECISION': ['min', 'mean'],\n    'CNT_PAYMENT': ['max', 'mean'],\n    'DAYS_TERMINATION': ['mean'],\n    # Engineered features\n    'CREDIT_TO_ANNUITY_RATIO': ['mean', 'max'],\n    'APPLICATION_CREDIT_DIFF': ['max'],\n    'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean'],\n    # The following features are only for approved applications\n    'DAYS_FIRST_DRAWING': ['max', 'mean'],\n    'DAYS_FIRST_DUE': ['min', 'mean'],\n    'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],\n    'DAYS_LAST_DUE': ['max', 'mean'],\n    'DAYS_LAST_DUE_DIFF': ['min', 'max', 'mean'],\n    'SIMPLE_INTERESTS': ['min', 'max', 'mean'],\n}\n\nPREVIOUS_REFUSED_AGG = {\n    'AMT_APPLICATION': ['max', 'mean'],\n    'AMT_CREDIT': ['min', 'max'],\n    'DAYS_DECISION': ['min', 'max', 'mean'],\n    'CNT_PAYMENT': ['max', 'mean'],\n    # Engineered features\n    'APPLICATION_CREDIT_DIFF': ['min', 'max', 'mean', 'var'],\n    'APPLICATION_CREDIT_RATIO': ['min', 'mean'],\n    'NAME_CONTRACT_TYPE_Consumer loans': ['mean'],\n    'NAME_CONTRACT_TYPE_Cash loans': ['mean'],\n    'NAME_CONTRACT_TYPE_Revolving loans': ['mean'],\n}\n\nPREVIOUS_LATE_PAYMENTS_AGG = {\n    'DAYS_DECISION': ['min', 'max', 'mean'],\n    'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],\n    # Engineered features\n    'APPLICATION_CREDIT_DIFF': ['min'],\n    'NAME_CONTRACT_TYPE_Consumer loans': ['mean'],\n    'NAME_CONTRACT_TYPE_Cash loans': ['mean'],\n    'NAME_CONTRACT_TYPE_Revolving loans': ['mean'],\n}\n\nPREVIOUS_LOAN_TYPE_AGG = {\n    'AMT_CREDIT': ['sum'],\n    'AMT_ANNUITY': ['mean', 'max'],\n    'SIMPLE_INTERESTS': ['min', 'mean', 'max', 'var'],\n    'APPLICATION_CREDIT_DIFF': ['min', 'var'],\n    'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean'],\n    'DAYS_DECISION': ['max'],\n    'DAYS_LAST_DUE_1ST_VERSION': ['max', 'mean'],\n    'CNT_PAYMENT': ['mean'],\n}\n\nPREVIOUS_TIME_AGG = {\n    'AMT_CREDIT': ['sum'],\n    'AMT_ANNUITY': ['mean', 'max'],\n    'SIMPLE_INTERESTS': ['mean', 'max'],\n    'DAYS_DECISION': ['min', 'mean'],\n    'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],\n    # Engineered features\n    'APPLICATION_CREDIT_DIFF': ['min'],\n    'APPLICATION_CREDIT_RATIO': ['min', 'max', 'mean'],\n    'NAME_CONTRACT_TYPE_Consumer loans': ['mean'],\n    'NAME_CONTRACT_TYPE_Cash loans': ['mean'],\n    'NAME_CONTRACT_TYPE_Revolving loans': ['mean'],\n}\n\nPOS_CASH_AGG = {\n    'SK_ID_PREV': ['nunique'],\n    'MONTHS_BALANCE': ['min', 'max', 'size'],\n    'SK_DPD': ['max', 'mean', 'sum', 'var'],\n    'SK_DPD_DEF': ['max', 'mean', 'sum'],\n    'LATE_PAYMENT': ['mean']\n}\n\nINSTALLMENTS_AGG = {\n    'SK_ID_PREV': ['size', 'nunique'],\n    'DAYS_ENTRY_PAYMENT': ['min', 'max', 'mean'],\n    'AMT_INSTALMENT': ['min', 'max', 'mean', 'sum'],\n    'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],\n    'DPD': ['max', 'mean', 'var'],\n    'DBD': ['max', 'mean', 'var'],\n    'PAYMENT_DIFFERENCE': ['mean'],\n    'PAYMENT_RATIO': ['mean'],\n    'LATE_PAYMENT': ['mean', 'sum'],\n    'SIGNIFICANT_LATE_PAYMENT': ['mean', 'sum'],\n    'LATE_PAYMENT_RATIO': ['mean'],\n    'DPD_7': ['mean'],\n    'DPD_15': ['mean'],\n    'PAID_OVER': ['mean']\n}\n\nINSTALLMENTS_TIME_AGG = {\n    'SK_ID_PREV': ['size'],\n    'DAYS_ENTRY_PAYMENT': ['min', 'max', 'mean'],\n    'AMT_INSTALMENT': ['min', 'max', 'mean', 'sum'],\n    'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],\n    'DPD': ['max', 'mean', 'var'],\n    'DBD': ['max', 'mean', 'var'],\n    'PAYMENT_DIFFERENCE': ['mean'],\n    'PAYMENT_RATIO': ['mean'],\n    'LATE_PAYMENT': ['mean'],\n    'SIGNIFICANT_LATE_PAYMENT': ['mean'],\n    'LATE_PAYMENT_RATIO': ['mean'],\n    'DPD_7': ['mean'],\n    'DPD_15': ['mean'],\n}\n\nCREDIT_CARD_AGG = {\n    'MONTHS_BALANCE': ['min'],\n    'AMT_BALANCE': ['max'],\n    'AMT_CREDIT_LIMIT_ACTUAL': ['max'],\n    'AMT_DRAWINGS_ATM_CURRENT': ['max', 'sum'],\n    'AMT_DRAWINGS_CURRENT': ['max', 'sum'],\n    'AMT_DRAWINGS_POS_CURRENT': ['max', 'sum'],\n    'AMT_INST_MIN_REGULARITY': ['max', 'mean'],\n    'AMT_PAYMENT_TOTAL_CURRENT': ['max', 'mean', 'sum', 'var'],\n    'AMT_TOTAL_RECEIVABLE': ['max', 'mean'],\n    'CNT_DRAWINGS_ATM_CURRENT': ['max', 'mean', 'sum'],\n    'CNT_DRAWINGS_CURRENT': ['max', 'mean', 'sum'],\n    'CNT_DRAWINGS_POS_CURRENT': ['mean'],\n    'SK_DPD': ['mean', 'max', 'sum'],\n    'SK_DPD_DEF': ['max', 'sum'],\n    'LIMIT_USE': ['max', 'mean'],\n    'PAYMENT_DIV_MIN': ['min', 'mean'],\n    'LATE_PAYMENT': ['max', 'sum'],\n}\n\nCREDIT_CARD_TIME_AGG = {\n    'CNT_DRAWINGS_ATM_CURRENT': ['mean'],\n    'SK_DPD': ['max', 'sum'],\n    'AMT_BALANCE': ['mean', 'max'],\n    'LIMIT_USE': ['max', 'mean']\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:53.448524Z","iopub.execute_input":"2022-08-04T06:42:53.448935Z","iopub.status.idle":"2022-08-04T06:42:53.487272Z","shell.execute_reply.started":"2022-08-04T06:42:53.448902Z","shell.execute_reply":"2022-08-04T06:42:53.486206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    pd.set_option('display.max_rows', 60)\n    pd.set_option('display.max_columns', 100)\n    with timer('Pipeline total time'):\n        main(debug = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T06:42:54.113126Z","iopub.execute_input":"2022-08-04T06:42:54.114398Z"},"trusted":true},"execution_count":null,"outputs":[]}]}