{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport pickle\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import KFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.metrics import roc_auc_score, roc_curve\n# from sklearn.utils.class_weight import compute_sample_weight\n\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\n# import xgboost as xgb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-14T20:16:52.683463Z","iopub.execute_input":"2024-05-14T20:16:52.683810Z","iopub.status.idle":"2024-05-14T20:16:59.569964Z","shell.execute_reply.started":"2024-05-14T20:16:52.683784Z","shell.execute_reply":"2024-05-14T20:16:59.569121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CAT_NAN_VALUE = '_NAN_VALUE_'\nCAT_RARE_VALUE = '_RARE_VALUE_'","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.571473Z","iopub.execute_input":"2024-05-14T20:16:59.571758Z","iopub.status.idle":"2024-05-14T20:16:59.575879Z","shell.execute_reply.started":"2024-05-14T20:16:59.571735Z","shell.execute_reply":"2024-05-14T20:16:59.574833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility functions","metadata":{}},{"cell_type":"code","source":"def compute_gini_stability_metric(base, w_fallingrate=88.0, w_resstd=-0.5, debug=False):\n    assert len(base.shape) == 2\n\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2 * roc_auc_score(x[\"target\"], x[\"score\"]) - 1).tolist()\n\n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a * x + b\n\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    score = avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\n    if debug:\n        print('a =', a, '; b =', b, '; res_std =', res_std, '; avg_gini =', avg_gini)\n        print('score =', score)\n        plt.figure(figsize=(16, 3))\n        plt.plot(x, y, marker='o', label='gini_in_time')\n        plt.plot(x, y_hat, marker='o', label='y_hat')\n        plt.ylim([0.4, 0.8])\n        plt.legend()\n\n    return score","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.577187Z","iopub.execute_input":"2024-05-14T20:16:59.577515Z","iopub.status.idle":"2024-05-14T20:16:59.606194Z","shell.execute_reply.started":"2024-05-14T20:16:59.577484Z","shell.execute_reply":"2024-05-14T20:16:59.605254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble model","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, ClassifierMixin):\n\n    def __init__(self, n_splits):\n        super().__init__()\n        self.n_splits = n_splits\n        self.estimators = []\n        self.eval_results = []\n        self.feature_importances = []\n        self.cv_results = []\n        self.feature_names = []\n\n    def validate_before_fit(self, X, y):\n        VotingModel._validate_X_y(X, y)\n        assert X.shape[0] >= 3\n        assert X.shape[1] >= 3\n        assert 'WEEK_NUM' in X.columns\n        assert 3 <= self.n_splits <= 100\n\n    def fit(self, X, y):\n        self.validate_before_fit(X, y)\n\n        self.estimators.clear()\n        self.eval_results.clear()\n        self.feature_importances.clear()\n        self.cv_results.clear()\n\n        week_num = X['WEEK_NUM']\n        x = X.drop(columns=['WEEK_NUM'])\n        self.feature_names = list(x.columns)\n\n        cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=False)\n        for idx_train, idx_valid in cv.split(x, y, groups=week_num):\n            X_train, y_train = x.iloc[idx_train], y.iloc[idx_train]\n            X_valid, y_valid = x.iloc[idx_valid], y.iloc[idx_valid]\n\n            # lgb_model = VotingModel._fit_lgb(X_train, y_train, X_valid, y_valid)\n            cat_model = VotingModel._fit_cat(X_train, y_train, X_valid, y_valid)\n            # self.estimators.append(lgb_model)\n            self.estimators.append(cat_model)\n   \n            # self.eval_results.append(VotingModel._get_lgb_eval(lgb_model))\n            self.eval_results.append(VotingModel._get_cat_eval(cat_model))\n            # self.feature_importances.append(VotingModel._get_lgb_fi(lgb_model))\n            self.feature_importances.append(VotingModel._get_cat_fi(cat_model))\n\n#             self.cv_results.append(VotingModel._construct_cv_result(\n#                 lgb_model, week_num, X_train, y_train, X_valid, y_valid\n#             ))\n            self.cv_results.append(VotingModel._construct_cv_result(\n                cat_model, week_num, X_train, y_train, X_valid, y_valid\n            ))\n\n        return self\n\n    def validate_before_predict(self, X):\n        assert len(X.shape) == 2\n        assert X.shape[0] >= 3\n        assert X.shape[1] >= 3\n        assert 'WEEK_NUM' in X.columns\n\n        assert len(self.estimators) == 1 * self.n_splits\n        assert len(self.eval_results) == 1 * self.n_splits\n        assert len(self.feature_importances) == 1 * self.n_splits\n        assert len(self.cv_results) == 1 * self.n_splits\n        assert len(self.feature_names) + 1 == X.shape[1]\n\n        x = X.drop(columns=['WEEK_NUM'])\n        assert x.shape[1] == len(self.feature_names)\n        for i in range(len(self.feature_names)):\n            assert self.feature_names[i] == x.columns[i]\n\n    def predict(self, X):\n        self.validate_before_predict(X)\n        raise ValueError('Not implemented.')\n\n        x = X.drop(columns=['WEEK_NUM'])\n        y_pred = np.mean(\n            [estimator.predict(x) for estimator in self.estimators],\n            axis=0\n        )\n        # y_pred = np.rint(y_pred)\n\n        assert y_pred.shape == (X.shape[0],)\n        return y_pred\n\n    def predict_proba(self, X):\n        self.validate_before_predict(X)\n\n        x = X.drop(columns=['WEEK_NUM'])\n        y_pred = np.mean(\n            [estimator.predict_proba(x) for estimator in self.estimators],\n            axis=0\n        )\n\n        assert y_pred.shape == (X.shape[0], 2)\n        return y_pred\n\n    @staticmethod\n    def _validate_X_y(X, y):\n        assert len(X.shape) == 2\n        assert len(y.shape) == 1\n        assert X.shape[0] == y.shape[0]\n        assert (X.index == y.index).all()\n\n    @staticmethod\n    def _fit_lgb(X_train, y_train, X_valid, y_valid):\n        VotingModel._validate_X_y(X_train, y_train)\n        VotingModel._validate_X_y(X_valid, y_valid)\n        assert 'WEEK_NUM' not in X_train.columns\n        assert 'WEEK_NUM' not in X_valid.columns\n\n        lgb_params = {\n            'objective': 'binary',\n            'boosting_type': 'gbdt',\n            'bagging_freq': 1,\n            'pos_bagging_fraction': 0.7,\n            'neg_bagging_fraction': 0.7,\n            'colsample_bynode': 0.8,\n            'colsample_bytree': 0.8,\n            'device': 'cpu',\n            # 'is_unbalance': True,\n            'learning_rate': 0.04,\n            'max_depth': 11,\n            'metric': 'auc',\n            # 'min_data_in_leaf': 30,\n            'n_estimators': 1000,\n            'num_leaves': 33,\n            'l1_regularization': 0.01,\n            'l2_regularization': 1.0,\n            'verbose': -1,\n        }\n\n        lgb_model = lgb.LGBMClassifier(**lgb_params)\n        lgb_model.fit(\n            X_train, y_train,\n            eval_set=[\n                (X_train, y_train),\n                (X_valid, y_valid),\n            ],\n            eval_metric=[\n                'auc',\n            ],\n            callbacks=[\n                lgb.log_evaluation(100),\n                lgb.early_stopping(100),\n            ]\n        )\n\n        return lgb_model\n\n    @staticmethod\n    def _fit_cat(X_train, y_train, X_valid, y_valid):\n        VotingModel._validate_X_y(X_train, y_train)\n        VotingModel._validate_X_y(X_valid, y_valid)\n        assert 'WEEK_NUM' not in X_train.columns\n        assert 'WEEK_NUM' not in X_valid.columns\n\n        cat_params = {\n            'eval_metric': 'AUC',\n            # 'learning_rate': 0.03,\n            'loss_function': 'Logloss',\n            # 'iterations': 900,\n            # 'depth': 8,\n            # 'max_ctr_complexity': 2,\n            # 'logging_level': 'Silent',\n            # 'plot': True,\n            # 'bootstrap_type': 'Bayesian',       \n            # 'bagging_temperature': 1,           \n            # 'od_type': 'Iter',                  \n            # 'od_wait': 50,\n            # 'task_type': 'GPU',\n            # 'devices': '0',\n        }\n\n        cat_features = []\n        for feature in X_train.columns:\n            if X_train[feature].dtype.name == 'category':\n                cat_features.append(feature)\n\n        cat_model = CatBoostClassifier(**cat_params)\n        cat_model.fit(\n            X_train, y_train,\n            eval_set=(X_valid, y_valid),\n            cat_features=cat_features,\n            use_best_model=True,\n            verbose=False\n        )\n\n        return cat_model\n    \n    @staticmethod\n    def _get_lgb_fi(lgb_model):\n        lgb_fi = lgb_model.feature_importances_\n        return lgb_fi / np.sum(lgb_fi) * 100.0\n\n    @staticmethod\n    def _get_cat_fi(cat_model):\n        cat_fi = cat_model.get_feature_importance()\n        assert np.isclose(np.sum(cat_fi), 100.0)\n        return cat_fi\n\n    @staticmethod\n    def _get_lgb_eval(lgb_model):\n        assert len(lgb_model.evals_result_) == 2\n        return {\n            'name': 'lgb',\n            'train_metric': lgb_model.evals_result_['training']['auc'],\n            'valid_metric': lgb_model.evals_result_['valid_1']['auc'],\n        }\n\n    @staticmethod\n    def _get_cat_eval(cat_model):\n        evals_result = cat_model.get_evals_result()\n        return {\n            'name': 'cat',\n            'train_loss': evals_result['learn']['Logloss'],\n            'valid_loss': evals_result['validation']['Logloss'],\n            'valid_metric': evals_result['validation']['AUC'],\n        }\n\n    @staticmethod\n    def _construct_cv_result(model, week_num,\n                             X_train, y_train, X_valid, y_valid):\n        VotingModel._validate_X_y(X_train, y_train)\n        VotingModel._validate_X_y(X_valid, y_valid)\n        assert model is not None and len(week_num.shape) == 1\n        assert week_num.shape[0] == X_train.shape[0] + X_valid.shape[0]\n        assert 'WEEK_NUM' not in X_train.columns\n        assert 'WEEK_NUM' not in X_valid.columns\n\n        y_pred_train = pd.Series(\n            model.predict_proba(X_train)[:, 1],\n            index=X_train.index\n        )\n        y_pred_valid = pd.Series(\n            model.predict_proba(X_valid)[:, 1],\n            index=X_valid.index\n        )\n\n        assert len(y_pred_train.shape) == len(y_pred_valid.shape) == 1\n        assert y_pred_train.shape[0] == X_train.shape[0]\n        assert y_pred_valid.shape[0] == X_valid.shape[0]\n\n        train_df = pd.DataFrame(data={\n            \"WEEK_NUM\": week_num.loc[y_train.index].to_numpy(),\n            \"target\": y_train.to_numpy(),\n            \"score\": y_pred_train.to_numpy()\n        })\n        valid_df = pd.DataFrame(data={\n            \"WEEK_NUM\": week_num.loc[y_valid.index].to_numpy(),\n            \"target\": y_valid.to_numpy(),\n            \"score\": y_pred_valid.to_numpy()\n        })\n \n        return {\n            'train_roc_auc_score': roc_auc_score(\n                y_true=y_train.to_numpy(), y_score=y_pred_train.to_numpy()\n            ),\n            'train_stability_score': compute_gini_stability_metric(\n                train_df, debug=False\n            ),\n            'valid_roc_auc_score': roc_auc_score(\n                y_true=y_valid.to_numpy(), y_score=y_pred_valid.to_numpy()\n            ),\n            'valid_stability_score': compute_gini_stability_metric(\n                valid_df, debug=True\n            ),\n            'train_df': train_df,\n            'valid_df': valid_df,\n            'y_valid': y_valid,\n            'y_pred_valid': y_pred_valid,\n        }","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.609070Z","iopub.execute_input":"2024-05-14T20:16:59.609639Z","iopub.status.idle":"2024-05-14T20:16:59.650132Z","shell.execute_reply.started":"2024-05-14T20:16:59.609613Z","shell.execute_reply":"2024-05-14T20:16:59.649263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pipeline","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int32))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n\n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.95:  # 0.7\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) and (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq <= 1) or (freq > 255):  # (freq <= 1) | (freq > 200)\n                    df = df.drop(col)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.651232Z","iopub.execute_input":"2024-05-14T20:16:59.651547Z","iopub.status.idle":"2024-05-14T20:16:59.664435Z","shell.execute_reply.started":"2024-05-14T20:16:59.651493Z","shell.execute_reply":"2024-05-14T20:16:59.663655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Automatic Aggregation","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n\n    @staticmethod\n    def _applicant_max_expr(df, depth, cols):\n        assert depth == 1\n        assert \"num_group1\" in df.columns\n        assert \"num_group2\" not in df.columns\n        assert \"num_group1\" not in cols\n\n        redundant_cols = [\n            'creationdate_885D',\n            'recorddate_4527225D',\n            'deductiondate_4917603D',\n            'debtoutstand_525A',\n            'debtoverdue_47A',\n            'empl_employedfrom_271D',\n            'empl_employedtotal_800L',\n            'empl_industry_691L',\n            'familystate_447L',\n            'housetype_905L',\n            'incometype_1044T',\n            'safeguarantyflag_411L',\n            'sex_738L',\n            'revolvingaccount_394A',\n            'residualamount_488A',\n            'instlamount_768A',\n            'instlamount_852A',\n            'outstandingamount_362A',\n            'totaldebtoverduevalue_178A',\n            'totaloutstanddebtvalue_39A',\n            'dateofcredstart_181D',\n            'numberofoverdueinstlmaxdat_641D',\n            'overdueamountmax2date_1142D',\n            'numberofcontrsvalue_258L',\n            'maxdpdtolerance_577P',\n            'outstandingdebt_522A',\n            'actualdpd_943P',\n            'financialinstitution_591M',\n            'contractst_545M',\n            'contractst_964M',\n        ]\n\n        return [\n            pl.col(\"num_group1\", col)\n            .filter(pl.col(\"num_group1\") == 0)\n            .exclude(\"num_group1\")\n            .max()\n            .alias(f\"applicant_max_{col}\")\n            for col in cols if col not in redundant_cols\n        ]\n\n    @staticmethod\n    def _above_expr(df, depth, cols, threshold):\n        assert depth in [1, 2]\n        assert \"num_group1\" in df.columns\n        return [\n            pl.col(col)\n            .filter(pl.col(col) > threshold)\n            .len()\n            .cast(pl.Float64)\n            .truediv(1 + pl.len())\n            .alias(f\"above_{threshold}_share_{col}\")\n            for col in cols\n        ]\n\n    @staticmethod\n    def _mode_share_expr(df, depth, cols):\n        assert depth in [1, 2]\n        assert \"num_group1\" in df.columns\n        \n        additional_param_cols = [\n            'numberofoverdueinstlmax_1151L',\n            'postype_4733339M',\n            'rejectreasonclient_4145042M',\n            'purposeofcred_874M',\n            'purposeofcred_426M',\n            'purposeofcred_722M',\n            'classificationofcontr_13M',\n            'contractst_545M',\n            'financialinstitution_591M',\n            'numberofoutstandinstls_520L',\n            'employername_M',\n            'name_4527232M',\n            'name_4917606M',\n            'employername_160M',\n            'numberofcontrsvalue_358L',\n            'contaddr_district_15M',\n            'registaddr_district_1083M',\n            'empladdr_district_926M',\n            'empladdr_zipcode_114M',\n        ]\n\n        expr = []\n\n        for col in cols:\n            if df[col].dtype == pl.String:\n                mode1 = pl.col(col).drop_nulls().mode().max()\n                # mode2 = pl.col(col).filter(pl.col(col) != mode1).mode().first()\n                expr.append(mode1.alias(f\"mode1_{col}\"))\n\n                if col in additional_param_cols:\n                    expr.append(\n                        (pl.col(col) == mode1).cast(pl.Float64).sum()\n                        .truediv(1 + pl.len())\n                        .alias(f\"mode1_additional_param_{col}\")\n                    )\n            else:\n                expr.append(\n                    pl.mean(col).alias(f\"mode1_{col}\")\n                )\n\n                if col in additional_param_cols:\n                    expr.append(\n                        pl.max(col).alias(f\"mode1_additional_param_{col}\")\n                    )\n\n        return expr\n\n    @staticmethod\n    def num_dpd_expr(df, depth):\n        assert depth in [1, 2]\n        \n        redundant_cols = [\n            'avgdbdtollast24m_4525197P',\n        ]\n\n        cols = [\n            col for col in df.columns\n            if (col[-1] == \"P\") and (col not in redundant_cols)\n        ]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n\n        expr_above = Aggregator._above_expr(\n            df,\n            depth,\n            [col for col in cols if 'pmts_dpd' in col],\n            threshold=0.0\n        )\n        \n        if depth == 2:\n            return expr_mean + expr_max + expr_above\n\n        expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n        return expr_mean + expr_max + expr_above + expr_applicant_max\n\n    @staticmethod\n    def num_amnt_expr(df, depth):\n        assert depth in [1, 2]\n        \n        redundant_cols = [\n            'avgpmtlast12m_4525200A',\n            'disbursedcredamount_1113A',\n            'maxpmtlast3m_4525190A',\n            'pmtamount_36A',\n            'totaloutstanddebtvalue_668A',\n            'mainoccupationinc_384A',\n            'credacc_credlmt_575A',\n        ]\n\n        cols = [\n            col for col in df.columns\n            if (col[-1] == \"A\") and (col not in redundant_cols)\n        ]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_min = [\n            pl.min(col).alias(f\"min_{col}\") for col in cols\n            if col not in [\n                'debtoutstand_525A',\n                'debtoverdue_47A',\n                'outstandingamount_362A',\n                'residualamount_488A',\n                'totalamount_996A',\n                'credlmt_935A',\n                'residualamount_856A',\n                'instlamount_768A',\n                \n            ]\n        ]\n\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_std = [\n            pl.std(col).alias(f\"std_{col}\")\n            for col in cols if 'amount' in col\n        ]\n\n        if depth == 2:\n            return expr_max + expr_min + expr_mean + expr_std\n\n        expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n        return expr_max + expr_min + expr_mean + expr_std + expr_applicant_max\n\n    @staticmethod\n    def date_expr(df, depth):\n        assert depth in [1, 2]\n\n        redundant_cols = [\n            'responsedate_1012D',\n            'processingdate_168D',\n            'numberofoverdueinstlmaxdat_148D',\n        ]\n\n        cols = [\n            col for col in df.columns\n            if (col[-1] == \"D\") and (col not in redundant_cols)\n        ]\n\n        expr_max = [\n            pl.max(col).alias(f\"max_{col}\") for col in cols\n            if col not in [\n                'recorddate_4527225D',\n                'approvaldate_319D',\n                'creationdate_885D',\n            ]\n        ]\n        expr_min = [\n            pl.min(col).alias(f\"min_{col}\") for col in cols\n            if col not in [\n                'empl_employedfrom_271D',\n                'recorddate_4527225D',\n            ]\n        ]\n\n        if depth == 1:\n            expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n            return expr_max + expr_min + expr_applicant_max\n\n        return expr_max + expr_min\n\n    @staticmethod\n    def str_expr(df, depth):\n        assert depth in [1, 2]\n\n        redundant_cols = [\n            'lastrejectcommodtypec_5251769M',\n            'language1_981M',\n            'subjectrole_182M',\n            'subjectrole_93M',\n            'subjectroles_name_838M',\n        ]\n\n        cols = [\n            col for col in df.columns\n            if (col[-1] == \"M\") and (col not in redundant_cols)\n        ]\n\n        expr_mode_share = Aggregator._mode_share_expr(df, depth, cols)\n\n        if depth == 2:\n            return expr_mode_share\n\n        expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n        return expr_mode_share + expr_applicant_max\n\n    @staticmethod\n    def other_expr(df, depth):\n        assert depth in [1, 2]\n        \n        redundant_cols = [\n            'pmts_month_158T',\n            'pmts_year_1139T',\n            'pmts_month_706T',\n            'dpdmaxdateyear_596T',\n            'overdueamountmaxdatemonth_284T',\n            'overdueamountmaxdatemonth_365T',\n            'dpdmaxdatemonth_89T',\n            'dpdmaxdatemonth_442T',\n\n            'contractssum_5085716L',\n            'applicationscnt_629L',\n            'clientscnt_1130L',\n            'clientscnt_360L',\n            'clientscnt_533L',\n            'clientscnt_887L',\n            'clientscnt_946L',\n            'numinstpaidlastcontr_4325080L',\n            'numnotactivated_1143L',\n            'contaddr_smempladdr_334L',\n            'type_25L',\n\n            # only 1 unique value\n            'personindex_1023L',\n            'persontype_1072L',\n            'persontype_792L',\n\n            # only 2 unique values\n            'contaddr_matchlist_1032L',  # [null, false]\n            'remitter_829L',             # [ 0. null]\n\n            # duplicates\n            'tenor_203L',  # ~ pmtnum_8L\n            \n            # score decrease\n            'periodicityofpmts_837L',\n            'overdueamountmaxdateyear_2T',\n            'numberofoutstandinstls_59L',\n            'annualeffectiverate_63L',\n            'pmtnum_8L',\n            'status_219L',\n        ]\n\n        cols = [\n            col for col in df.columns\n            if (col[-1] in (\"T\", \"L\")) and (col not in redundant_cols)\n        ]\n\n        expr_mode_share = Aggregator._mode_share_expr(df, depth, cols)\n\n        if depth == 2:\n            return expr_mode_share\n\n        expr_applicant_max = Aggregator._applicant_max_expr(df, depth, cols)\n        return expr_mode_share + expr_applicant_max\n    \n    @staticmethod\n    def count_expr(df, depth):\n        assert depth in [1, 2]\n\n        cols = [\n            col for col in df.columns\n            if \"num_group\" in col\n        ]\n\n        if depth == 1:\n            assert \"num_group1\" in df.columns\n            assert \"num_group2\" not in df.columns\n            expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n            return expr_max\n\n        assert \"num_group1\" in df.columns\n        assert \"num_group2\" in df.columns\n\n        return []  # expr_n_unique + expr_applicant_share\n\n    @staticmethod\n    def get_exprs(df, depth):\n        assert depth in [1, 2]\n\n        exprs = Aggregator.num_dpd_expr(df, depth) + \\\n                Aggregator.num_amnt_expr(df, depth) + \\\n                Aggregator.date_expr(df, depth) + \\\n                Aggregator.str_expr(df, depth) + \\\n                Aggregator.other_expr(df, depth) + \\\n                Aggregator.count_expr(df, depth)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.665587Z","iopub.execute_input":"2024-05-14T20:16:59.665863Z","iopub.status.idle":"2024-05-14T20:16:59.703951Z","shell.execute_reply.started":"2024-05-14T20:16:59.665841Z","shell.execute_reply":"2024-05-14T20:16:59.703038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### File I/O","metadata":{}},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df, depth))\n\n    return df\n\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df, depth))\n        \n        chunks.append(df)\n\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.705058Z","iopub.execute_input":"2024-05-14T20:16:59.705353Z","iopub.status.idle":"2024-05-14T20:16:59.717987Z","shell.execute_reply.started":"2024-05-14T20:16:59.705329Z","shell.execute_reply":"2024-05-14T20:16:59.717062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base.with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n\n    df_base = df_base.pipe(Pipeline.handle_dates)\n\n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.718943Z","iopub.execute_input":"2024-05-14T20:16:59.719197Z","iopub.status.idle":"2024-05-14T20:16:59.730392Z","shell.execute_reply.started":"2024-05-14T20:16:59.719175Z","shell.execute_reply":"2024-05-14T20:16:59.729459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n\n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.731457Z","iopub.execute_input":"2024-05-14T20:16:59.732271Z","iopub.status.idle":"2024-05-14T20:16:59.740580Z","shell.execute_reply.started":"2024-05-14T20:16:59.732245Z","shell.execute_reply":"2024-05-14T20:16:59.739820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.744750Z","iopub.execute_input":"2024-05-14T20:16:59.745147Z","iopub.status.idle":"2024-05-14T20:16:59.749899Z","shell.execute_reply.started":"2024-05-14T20:16:59.745117Z","shell.execute_reply":"2024-05-14T20:16:59.749080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Files Read & Feature Engineering","metadata":{}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:16:59.751103Z","iopub.execute_input":"2024-05-14T20:16:59.752016Z","iopub.status.idle":"2024-05-14T20:22:44.983164Z","shell.execute_reply.started":"2024-05-14T20:16:59.751989Z","shell.execute_reply":"2024-05-14T20:22:44.982104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:22:44.984357Z","iopub.execute_input":"2024-05-14T20:22:44.984659Z","iopub.status.idle":"2024-05-14T20:23:05.249349Z","shell.execute_reply.started":"2024-05-14T20:22:44.984636Z","shell.execute_reply":"2024-05-14T20:23:05.248435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:23:05.250730Z","iopub.execute_input":"2024-05-14T20:23:05.251077Z","iopub.status.idle":"2024-05-14T20:23:05.829597Z","shell.execute_reply.started":"2024-05-14T20:23:05.251047Z","shell.execute_reply":"2024-05-14T20:23:05.828579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.pipe(Pipeline.filter_cols)\n\nassert len(df_train.shape) == 2\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:23:05.830865Z","iopub.execute_input":"2024-05-14T20:23:05.831195Z","iopub.status.idle":"2024-05-14T20:23:10.126079Z","shell.execute_reply.started":"2024-05-14T20:23:05.831172Z","shell.execute_reply":"2024-05-14T20:23:10.125162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, cat_cols = to_pandas(df_train)\ntrain_cols = list(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:23:10.127212Z","iopub.execute_input":"2024-05-14T20:23:10.127498Z","iopub.status.idle":"2024-05-14T20:23:31.618266Z","shell.execute_reply.started":"2024-05-14T20:23:10.127474Z","shell.execute_reply":"2024-05-14T20:23:31.617439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessor","metadata":{}},{"cell_type":"code","source":"class Preprocessor:\n\n    def __init__(self):\n        self._cat_col2valid_values = dict()\n\n    def fit(self, X):\n        assert len(X.shape) == 2\n        assert X.shape[0] >= 2\n\n        self._cat_col2valid_values.clear()\n        for col in X.columns:\n            if X[col].dtype.name == 'category':\n                value_counts = X[col].value_counts(normalize=True, sort=True)\n                max_unique_values_n = 20\n                assert max_unique_values_n >= 2\n                valid_values = value_counts.index[:max_unique_values_n]\n                valid_values = set(valid_values.tolist())\n                valid_values = valid_values.union({CAT_NAN_VALUE, CAT_RARE_VALUE})\n                self._cat_col2valid_values[col] = list(valid_values)\n\n        return self\n\n    def transform(self, X):\n        assert len(X.shape) == 2\n        assert X.shape[0] >= 1\n\n        out_df = X.copy()\n        out_df = out_df.set_index('case_id')\n\n        target = None\n        if 'target' in out_df.columns:\n            out_df = out_df.sort_values(by=['WEEK_NUM', 'case_id'])\n            target = out_df['target']\n            out_df = out_df.drop(columns=['target'])\n\n        # ============================================================\n        # Preprocess categorical features\n        for cat_col, valid_values in self._cat_col2valid_values.items():\n            assert out_df[cat_col].dtype.name == 'category'\n            out_df[cat_col] = out_df[cat_col].cat.add_categories([\n                CAT_NAN_VALUE, CAT_RARE_VALUE\n            ])\n            out_df[cat_col] = out_df[cat_col].fillna(CAT_NAN_VALUE)\n            mask = ~out_df[cat_col].isin(valid_values)\n            out_df.loc[mask, cat_col] = CAT_RARE_VALUE\n            out_df[cat_col] = out_df[cat_col].cat.set_categories(valid_values)\n            assert out_df[cat_col].dtype.name == 'category'\n        # ============================================================\n\n        # ============================================================\n        # drop redundant columns\n        out_df.drop(columns=Preprocessor._get_redundant_columns(), inplace=True)\n        # ============================================================\n\n        # ============================================================\n        # aggregate some columns\n        agg2group = Preprocessor._get_agg2group()\n        agg_df = Preprocessor._get_agg_df(out_df, agg2group)\n\n        mode2group = Preprocessor._get_mode2group()\n        mode_df = Preprocessor._get_mode_df(out_df, mode2group)\n\n        out_df.drop(\n            columns=Preprocessor._get_groups_to_drop(agg2group, mode2group),\n            inplace=True\n        )\n        out_df = pd.concat([agg_df, mode_df, out_df], axis=1)\n        # ============================================================\n\n        # ============================================================\n        # construct new features (sometimes, e.g.\n        # feature1 - feature2 > 0 => higher chance of default)\n        mix_df = pd.concat([\n            Preprocessor._get_add_df(out_df, Preprocessor._get_add_pairs()),\n            Preprocessor._get_sub_df(out_df, Preprocessor._get_sub_pairs()),\n            Preprocessor._get_mul_df(out_df, Preprocessor._get_mul_pairs()),\n            Preprocessor._get_div_df(out_df, Preprocessor._get_div_pairs()),\n            # Preprocessor._get_compound_df(out_df, agg_df),\n        ], axis=1)\n\n        out_df = pd.concat([mix_df, out_df], axis=1)\n\n        # ============================================================\n        # select columns\n        out_df = out_df[\n            ['WEEK_NUM']\n            + Preprocessor._get_primary_features()\n            + Preprocessor._get_secondary_features()\n        ]\n        # ============================================================\n\n        # ============================================================\n        # handle NaNs - fillna?\n        # out_df['missing_values_n'] = out_df.isna().sum(axis=1)\n        # ============================================================\n\n        return out_df, target\n    \n    @staticmethod\n    def _get_redundant_columns():\n        return [\n            'deferredmnthsnum_166L',         # only 1 unique value\n            'commnoinclast6m_3546845L',      # only 0 and nan\n            'mastercontrelectronic_519L',    # only 0 and nan\n            'mastercontrexist_109L',         # [ 0. nan]\n            'bankacctype_710L',              # [NaN, 'CA']\n            'isdebitcard_729L',              # [NaN, False]\n            'paytype1st_925L',               # ['OTHER', NaN]\n            'paytype_783L',                  # ['OTHER', NaN]\n            'typesuite_864L',                # [NaN, 'AL'] \n            # 'mode1_subjectrole_182M',        # [NaN, 'a55475b1']\n            # 'mode1_subjectrole_93M',         # [NaN, 'a55475b1']\n            # 'mode1_subjectroles_name_838M',  # [NaN, 'a55475b1']\n\n            # duplicated columns\n            'interestrate_311L',\n            'paytype_783L',\n            'mean_debtoutstand_525A',\n            \n            # other columns contain the same values, but have less nans\n            'assignmentdate_4527235D',\n\n            # score decrease\n            'month_decision',\n            'weekday_decision',\n            'firstclxcampaign_1125D',\n\n            'firstquarter_103L',\n            'secondquarter_766L',\n            # 'thirdquarter_1082L',\n            'fourthquarter_440L',\n            'max_num_group1_5',\n            'max_num_group1_6',\n            'max_num_group1',\n            'max_num_group1_9',\n            'max_num_group1_10',\n            'max_num_group1_11',\n\n            'pmtcount_4527229L',\n            'pmtcount_693L',\n            'pmtscount_423L',\n            'pmtssum_45A',\n            # 'assignmentdate_238D',\n            # 'responsedate_1012D',\n            'applicationscnt_464L',\n            'applicationscnt_629L',\n        ]\n\n    @staticmethod\n    def _get_mode2group():\n        return {\n            'agg_applicant_education_M': [\n                'education_1103M',\n                'applicant_max_education_1138M',\n                'education_88M',\n                'applicant_max_education_927M',\n            ],\n        }\n    \n    @staticmethod\n    def _get_mode_df(df, mode2group):\n        assert len(df.shape) == 2\n        assert type(mode2group) is dict\n        assert len(mode2group) == len(Preprocessor._get_mode2group())\n        \n        mode_data = dict()\n        for mode, group in mode2group.items():\n            mode_data[mode] = df[group].mode(axis=1).iloc[:, 0].astype('category')\n\n        return pd.DataFrame(index=df.index, data=mode_data).astype('category')\n\n    @staticmethod\n    def _get_agg2group():\n        # Return aggregation feature name -> group of features\n        agg2group = {\n            'agg_birth_D': [\n                'birthdate_574D', 'dateofbirth_337D',\n                'applicant_max_birth_259D',\n                'max_birth_259D', 'min_birth_259D',\n            ],\n            'agg_credit_bureau_queries_L': [\n                'days120_123L', 'days180_256L', 'days30_165L',\n                'days360_512L', 'days90_310L',\n            ],\n            'agg_avgdbddpdlast24m_P': [\n                'avgdpdtolclosure24_3658938P',\n                'avgdbddpdlast24m_3658932P',\n                # 'avgdbddpdlast3m_4187120P',\n                'avgdbdtollast24m_4525197P',\n            ],\n            'agg_phone_number_shares_L': [\n                'clientscnt_100L', 'clientscnt_1022L',\n                'clientscnt_1071L', 'clientscnt_1130L',\n                'clientscnt_157L', 'clientscnt_257L',\n                'clientscnt_304L', 'clientscnt_360L',\n                'clientscnt_493L', 'clientscnt_533L',\n                'clientscnt_887L', 'clientscnt_946L',\n                'clientscnt12m_3712952L',\n                'clientscnt3m_3712950L',\n                'clientscnt6m_3712949L',\n            ],\n            'agg_applicant_dtlastpmt_D': [\n                'applicant_max_dtlastpmt_581D',\n                'applicant_max_dtlastpmtallstes_3545839D',\n            ],\n            'agg_min_dtlastpmt_D': [\n                'min_dtlastpmt_581D',\n                'min_dtlastpmtallstes_3545839D',\n            ],\n            'agg_maxdbddpd_P': [  # diffs are interesting\n                'maxdbddpdlast1m_3658939P',\n                'maxdbddpdtollast12m_3658940P',\n                'maxdbddpdtollast6m_4187119P',\n            ],\n            'agg_maxdpdlast_P': [  # diffs are interesting\n                'maxdpdlast12m_727P',\n                'maxdpdlast24m_143P',\n                'maxdpdlast3m_392P',\n                'maxdpdlast6m_474P',\n                'maxdpdlast9m_1059P',\n            ],\n            'agg_mindpdlast24m_P': [\n                'mindbddpdlast24m_3658935P',\n                'mindbdtollast24m_4525191P',\n            ],\n            'agg_numactivecreds_L': [\n                'numactivecreds_622L',\n                'numactivecredschannel_414L',\n            ],\n            # 'agg_pmtcount_L': [\n            #     'pmtcount_4527229L',\n            #     'pmtcount_693L',\n            #     'pmtscount_423L',\n            # ],\n            'agg_pmtaverage_A': [\n                'pmtaverage_3A',\n                'pmtaverage_4527227A',\n            ],\n            'agg_numinstpaid_L': [\n                'numinstlsallpaid_934L',\n                'numinstpaid_4499208L',\n            ],\n            'agg_numinstnotpaid_L': [\n                'numinsttopaygr_769L',\n                'numinsttopaygrest_4493213L',\n            ],\n            'agg_numinstpaidearlymix_L': [\n                'numinstpaidearly3d_3546850L',\n                'numinstpaidearly3dest_4493216L',\n                'numinstmatpaidtearly2d_4499204L',\n                'numinstlallpaidearly3d_817L',\n                'numinstpaidearly5dobd_4499205L',\n                'numinstpaidearly_338L',\n                'numinstpaidearlyest_4493214L',\n            ],\n            'agg_numinstpaidearly5d_L': [\n                'numinstpaidearly5d_1087L',\n                'numinstpaidearly5dest_4493211L',\n            ],\n            'agg_numinstregularpaid_L': [\n                'numinstregularpaid_973L',\n                'numinstregularpaidest_4493210L',\n            ],\n            'agg_numinstunpaidmax_L': [\n                'numinstunpaidmax_3546851L',\n                'numinstunpaidmaxest_4493212L',\n            ],\n            'agg_pctinstlsallpaidlat_L': [\n                'pctinstlsallpaidlat10d_839L',\n                'pctinstlsallpaidlate1d_3546856L',\n                'pctinstlsallpaidlate4d_3546849L',\n                'pctinstlsallpaidlate6d_3546844L',\n            ],\n\n            # correlation is very close to 1 (0.9999...)\n            'agg_dpd_P': [\n                'actualdpdtolerance_344P',\n                'max_actualdpd_943P',\n            ],\n            'agg_currtotaldebt_A': [\n                'currdebt_22A',\n                'totaldebt_9A',\n            ],\n            'agg_activateddate_dateactivated_D': [\n                'lastactivateddate_801D',\n                'max_dateactivated_425D',\n                'applicant_max_dateactivated_425D',\n            ],\n            'agg_apprdate_D': [\n                'lastapprdate_640D',\n                'applicant_max_approvaldate_319D',\n            ],\n            'agg_sumoutstandtotal_A': [\n                'sumoutstandtotal_3546847A',\n                'sumoutstandtotalest_4493215A',\n            ],\n            'agg_firstnonzeroinstldate_D': [\n                'max_firstnonzeroinstldate_307D',\n                'applicant_max_firstnonzeroinstldate_307D',\n            ],\n            'agg_dpdmax_139_pmts_dpd_1073_P': [\n                'max_dpdmax_139P',\n                'max_pmts_dpd_1073P',\n            ],\n            'agg_dpdmax_757_pmts_dpd_303_P': [\n                'max_dpdmax_757P',\n                'max_pmts_dpd_303P',\n            ],\n            'agg_totaldebtoverduevalue_A': [\n                'max_overdueamount_31A',\n                'max_totaldebtoverduevalue_718A',\n                'mean_totaldebtoverduevalue_718A',\n                'min_totaldebtoverduevalue_718A',\n                'applicant_max_totaldebtoverduevalue_718A',\n            ],\n            'agg_numberofoutstandinstls_L': [\n                'mode1_numberofoutstandinstls_520L',\n                'mode1_additional_param_numberofoutstandinstls_520L',\n            ],\n\n            # correlation analysis (correlations 0.95+)\n            'agg_dtlastpmtallstes_D': [\n                'dtlastpmtallstes_4499206D',\n                'max_dtlastpmtallstes_3545839D',\n            ],\n            'agg_approvaldate_firstnonzeroinstldate_D': [\n                'lastapplicationdate_877D',\n                'applicant_max_approvaldate_319D',\n                'applicant_max_dateactivated_425D',\n                'lastapprdate_640D',\n                'applicant_max_firstnonzeroinstldate_307D',\n            ],\n            'agg_min_approvaldate_dateactivated_D': [\n                'min_approvaldate_319D',\n                'min_dateactivated_425D',\n            ],\n            'agg_creationdate_firstnonzeroinstldate_D': [\n                'min_creationdate_885D',\n                'min_firstnonzeroinstldate_307D',\n            ],\n            'agg_max_amount_A': [\n                'max_amount_4527230A',\n                'max_amount_4917619A',\n                'applicant_max_amount_4527230A',\n                'applicant_max_amount_4917619A',\n            ],\n            'agg_min_amount_A': [\n                'min_amount_4527230A',\n                'min_amount_4917619A',\n            ],\n            'agg_mean_amount_A': [\n                'mean_amount_4527230A',\n                'mean_amount_4917619A',\n            ],\n            'agg_std_amount_A': [\n                'std_amount_4527230A',\n                'std_amount_4917619A',\n            ],\n            'agg_mode1_additional_param_employername_M': [\n                'mode1_additional_param_name_4527232M',\n                'mode1_additional_param_name_4917606M',\n                'mode1_additional_param_employername_160M',\n            ],\n            'agg_max_num_group1_3_4': [\n                'max_num_group1_3',\n                'max_num_group1_4',\n            ],\n            'agg_totaldebtoverdue_A': [\n                'max_debtoverdue_47A',\n                'max_overdueamount_659A',\n                'max_totaldebtoverduevalue_178A',\n                'mean_totaldebtoverduevalue_178A',\n            ],\n            'agg_applicant_credlmt_totalamount_A': [\n                'applicant_max_credlmt_230A',\n                'applicant_max_totalamount_6A',\n            ],\n            'agg_overdueamountmaxdateyear_T': [\n                'mode1_dpdmaxdateyear_896T',\n                'mode1_overdueamountmaxdateyear_994T',\n            ],\n            'agg_numberofcontrsvalue_L': [\n                'mode1_additional_param_numberofcontrsvalue_358L',\n                'applicant_max_numberofcontrsvalue_358L',\n                'mode1_numberofcontrsvalue_358L',\n            ],\n            'agg_applicant_overdueamountmaxdateyear_T': [\n                'applicant_max_dpdmaxdateyear_896T',\n                'applicant_max_overdueamountmaxdateyear_994T',\n            ],\n            'agg_mode1_additional_param_contaddr_registaddr_district_M': [\n                'mode1_additional_param_contaddr_district_15M',\n                'mode1_additional_param_registaddr_district_1083M',\n            ],\n            'agg_mode1_additional_param_empladdr_district_zipcode_M': [\n                'mode1_additional_param_empladdr_district_926M',\n                'mode1_additional_param_empladdr_zipcode_114M',\n            ],\n            'agg_max_openingdate_D': [\n                'max_openingdate_313D',\n                'max_openingdate_857D',\n            ],\n            'agg_min_openingdate_D': [\n                'min_openingdate_313D',\n                'min_openingdate_857D',\n            ],\n            'agg_applicant_openingdate_D': [\n                'applicant_max_openingdate_313D',\n                'applicant_max_openingdate_857D',\n            ],\n            'agg_mode1_additional_param_13_545_591_426_M': [\n                'mode1_additional_param_classificationofcontr_13M',\n                'mode1_additional_param_contractst_545M',\n                'mode1_additional_param_financialinstitution_591M',\n                'mode1_additional_param_purposeofcred_426M',\n            ],\n        }\n\n        for prefix in ['max', 'min', 'mean', 'std', 'applicant_max']:\n            agg2group['agg_' + prefix + '_overdueamountmax_14155A'] = [\n                prefix + '_overdueamountmax2_14A',\n                prefix + '_overdueamountmax_155A',\n            ]\n            agg2group['agg_' + prefix + '_overdueamountmax_35398A'] = [\n                prefix + '_overdueamountmax2_398A',\n                prefix + '_overdueamountmax_35A',\n            ]\n            \n        return agg2group\n    \n    @staticmethod\n    def _get_agg_df(df, agg2group):\n        assert len(df.shape) == 2\n        assert type(agg2group) is dict\n        assert len(agg2group) == len(Preprocessor._get_agg2group())\n\n        agg_data = dict()\n        for agg, group in agg2group.items():\n            agg_data[agg] = df[group].mean(axis=1)\n\n        return pd.DataFrame(index=df.index, data=agg_data)\n\n    @staticmethod\n    def _get_add_pairs():\n        return [\n            ['agg_currtotaldebt_A', 'agg_sumoutstandtotal_A', 1.4, 1.0],\n            [\n                'mode1_numberofoverdueinstlmax_1039L',\n                'mode1_numberofoverdueinstlmax_1151L',\n                4.0, 1.0\n            ],\n            # ['mean_dpdmax_757P', 'maxdpdlast24m_143P', 0.35, 1.0],\n            # ['mean_pmts_dpd_303P', 'maxdpdlast24m_143P', 0.38, 1.0],\n            ['mean_pmts_overdue_1140A', 'mean_pmts_overdue_1152A', 3.0, 1.0],\n        ]\n    \n    @staticmethod\n    def _get_add_df(df, add_pairs):\n        assert len(df.shape) == 2\n        assert type(add_pairs) is list\n        assert len(add_pairs) == len(Preprocessor._get_add_pairs())\n        \n        for add_pair in add_pairs:\n            assert len(add_pair) == 4\n\n        add_data = dict()\n        for (f1, f2, w1, w2) in add_pairs:\n            feature_name = '_'.join(['add', f1, f2])\n            add_data[feature_name] = w1 * df[f1] + w2 * df[f2]\n        \n        return pd.DataFrame(index=df.index, data=add_data)\n\n    @staticmethod\n    def _get_sub_pairs():\n        return [\n            ['agg_currtotaldebt_A', 'max_outstandingdebt_522A', 1.27, 1.0],\n            # ['agg_totaldebtoverdue_A', 'mean_totaldebtoverduevalue_178A', 1.0, 1.0],\n            # ['agg_totaldebtoverdue_A', 'max_pmts_overdue_1140A', 2.0, 1.0],\n            ['agg_totaldebtoverdue_A', 'agg_max_overdueamountmax_14155A', 2.0, 1.0],\n            ['mean_totaldebtoverduevalue_178A', 'mean_pmts_overdue_1140A', 0.65, 1.0],\n            ['numincomingpmts_3546848L', 'numinstlswithdpd10_728L', 0.11, 1.0],\n            \n            # ['min_annuity_853A', 'min_totaldebtoverduevalue_178A', 0.75, 1.0],\n            ['numinstlswithdpd5_4187116L', 'numinstlswithoutdpd_562L', 14.1, 1.0],\n            # ['datelastinstal40dpd_247D', 'max_dateofcredstart_181D', 0.42, 1.0],\n            ['agg_min_overdueamountmax_35398A', 'min_annuity_853A', 1.3, 1.0],\n            # ['agg_totaldebtoverdue_A', 'mean_annuity_853A', 1.65, 1.0],\n            ['annuity_780A', 'mean_annuity_853A', 0.83, 1.0],\n            ['annuity_780A', 'applicant_max_annuity_853A', 0.9, 1.0],\n            ['agg_avgdbddpdlast24m_P', 'avgdbddpdlast3m_4187120P', 0.65, 1.0],\n            ['maxdbddpdlast1m_3658939P', 'maxdbddpdtollast12m_3658940P', 1.17, 1.0],\n            ['mean_overdueamount_659A', 'mean_overdueamount_31A', 0.02, 1.0],\n            # ['maxdpdlast12m_727P', 'maxdpdlast24m_143P', 1.6, 1.0],\n            # ['maxdpdlast9m_1059P', 'maxdpdlast12m_727P', 1.24, 1.0],\n            # ['maxdpdlast6m_474P', 'maxdpdlast9m_1059P', 1.33, 1.0],\n            ['maxdpdlast3m_392P', 'maxdpdlast6m_474P', 1.5, 1.0],\n            # ['maxdpdlast3m_392P', 'maxdpdlast24m_143P', 4.0, 1.0],\n            # ['currdebt_22A', 'currdebt_94A', 1.0, 1.0],\n            ['maininc_215A', 'mean_mainoccupationinc_437A', 0.83, 1.0],\n            # ['mean_dpdmax_139P', 'mean_dpdmax_757P', 4.5, 1.0],\n            ['mean_monthlyinstlamount_332A', 'mean_monthlyinstlamount_674A', 1.2, 1.0],\n            ['mean_totalamount_996A', 'mean_totalamount_6A', 0.35, 1.0],\n        ]\n\n    @staticmethod\n    def _get_sub_df(df, sub_pairs):\n        assert len(df.shape) == 2\n        assert type(sub_pairs) is list\n        assert len(sub_pairs) == len(Preprocessor._get_sub_pairs())\n        \n        for sub_pair in sub_pairs:\n            assert len(sub_pair) == 4\n\n        sub_data = dict()\n        for (f1, f2, w1, w2) in sub_pairs:\n            feature_name = '_'.join(['sub', f1, f2])\n            sub_data[feature_name] = w1 * df[f1] - w2 * df[f2]\n\n        return pd.DataFrame(index=df.index, data=sub_data)\n    \n    @staticmethod\n    def _get_mul_pairs():\n        return [\n            # ['maxdpdlast12m_727P', 'agg_sumoutstandtotal_A'],\n            # ['agg_sumoutstandtotal_A', 'applicant_max_dateofcredend_289D'],\n            # ['mean_residualamount_488A', 'mean_pmts_overdue_1152A'],\n        ]\n    \n    @staticmethod\n    def _get_mul_df(df, mul_pairs):\n        assert len(df.shape) == 2\n        assert type(mul_pairs) is list\n        assert len(mul_pairs) == len(Preprocessor._get_mul_pairs())\n        \n        for mul_pair in mul_pairs:\n            assert len(mul_pair) == 2\n\n        mul_data = dict()\n        for (f1, f2) in mul_pairs:\n            feature_name = '_'.join(['mul', f1, f2])\n            mul_data[feature_name] = df[f1] * df[f2]\n\n        return pd.DataFrame(index=df.index, data=mul_data)\n\n    @staticmethod\n    def _get_div_pairs():\n        return [\n            ['annuity_780A', 'maininc_215A'],\n            ['agg_currtotaldebt_A', 'maininc_215A'],\n            ['mean_debtoverdue_47A', 'maininc_215A'],\n            ['mean_totaloutstanddebtvalue_39A', 'maininc_215A'],\n            ['avglnamtstart24m_4525187A', 'maininc_215A'],\n            ['avgpmtlast12m_4525200A', 'maininc_215A'],\n            ['agg_sumoutstandtotal_A', 'maininc_215A'],\n            ['maxdpdlast12m_727P', 'maininc_215A'],\n            ['maxpmtlast3m_4525190A', 'maininc_215A'],\n            ['inittransactionamount_650A', 'price_1097A'],\n            ['agg_sumoutstandtotal_A', 'price_1097A'],\n            ['agg_currtotaldebt_A', 'price_1097A'],\n            # ['maxpmtlast3m_4525190A', 'price_1097A'],\n            ['mean_credacc_actualbalance_314A', 'agg_numactivecreds_L'],\n            ['mean_credacc_actualbalance_314A', 'avgpmtlast12m_4525200A'],\n            ['agg_currtotaldebt_A', 'applicant_max_dateofcredend_289D'],\n            # ['mean_debtoverdue_47A', 'applicant_max_dateofcredend_289D'],\n            # ['agg_sumoutstandtotal_A', 'applicant_max_dateofcredend_289D'],\n            ['maininc_215A', 'applicant_max_childnum_21L'],\n            ['mean_monthlyinstlamount_332A', 'inittransactionamount_650A'],\n            ['mean_monthlyinstlamount_332A', 'maininc_215A'],\n            ['mean_totaldebtoverduevalue_178A', 'maininc_215A'],\n        ]\n    \n    @staticmethod\n    def _get_div_df(df, div_pairs):\n        assert len(df.shape) == 2\n        assert type(div_pairs) is list\n        assert len(div_pairs) == len(Preprocessor._get_div_pairs())\n        \n        for div_pair in div_pairs:\n            assert len(div_pair) == 2\n\n        div_data = dict()\n        for (f1, f2) in div_pairs:\n            feature_name = '_'.join(['div', f1, f2])\n            div_data[feature_name] = df[f1] / (1.0 + df[f2])\n\n        return pd.DataFrame(index=df.index, data=div_data)\n    \n#     @staticmethod\n#     def _get_compound_df(df):\n#         assert len(df.shape) == 2\n\n#         compound_data = {\n#             'compound_coef_1_DA': (\n#                 df['mean_monthlyinstlamount_332A']\n#                 / (1.0 + df['maininc_215A'])\n#                 * df['applicant_max_dateofcredend_289D']\n#             ),\n#             'compound_coef_2_DA': (\n#                 df['maininc_215A']\n#                 * df['applicant_max_dateofcredend_289D']\n#                 / (1.0 + df['price_1097A'])\n#             ),\n#             'compound_coef_3_DA': (\n#                 df['maininc_215A']\n#                 * df['applicant_max_dateofcredend_289D']\n#                 - df['agg_sumoutstandtotal_A']\n#             ),\n#             'compound_coef_4_DA': (\n#                 df['maininc_215A'] * df['applicant_max_dateofcredend_289D']\n#                 + df['mean_credacc_actualbalance_314A']\n#                 - df['agg_sumoutstandtotal_A']\n#             ) / (1.0 + df['agg_numactivecreds_L']),\n#         }\n\n#         return pd.DataFrame(index=df.index, data=compound_data)\n\n    @staticmethod\n    def _get_groups_to_drop(agg2group, mode2group):\n        assert type(agg2group) is dict\n        assert type(mode2group) is dict\n        assert len(agg2group) == len(Preprocessor._get_agg2group())\n        assert len(mode2group) == len(Preprocessor._get_mode2group())\n\n        features_to_drop = []\n        features_to_keep = [\n            'mean_totaldebtoverduevalue_178A',\n            'maxdbddpdlast1m_3658939P',\n            'maxdbddpdtollast12m_3658940P',\n            'maxdpdlast12m_727P',\n            'maxdpdlast24m_143P',\n            'maxdpdlast9m_1059P',\n            'maxdpdlast6m_474P',\n            'maxdpdlast3m_392P',\n        ]\n\n        for _, group in agg2group.items():\n            for feature in group:\n                if feature not in features_to_keep:\n                    features_to_drop.append(feature)\n\n        for _, group in mode2group.items():\n            for feature in group:\n                if feature not in features_to_keep:\n                    features_to_drop.append(feature)\n\n        return features_to_drop\n\n    @staticmethod\n    def _get_primary_features():\n        return [\n            'annuity_780A',\n            'pmtnum_254L',\n            'eir_270L',\n            'mobilephncnt_593L',\n            'price_1097A',\n\n            'agg_birth_D',\n            'requesttype_4525192L',\n            'amtinstpaidbefduel24m_4187115A',\n            'annuitynextmonth_57A',\n            'agg_avgdbddpdlast24m_P',\n\n            'mean_pmts_dpd_1073P',\n            'mean_pmts_overdue_1140A',\n            'applicant_max_dpdmax_139P',\n            'above_0.0_share_pmts_dpd_1073P',\n            'lastrejectdate_50D',\n\n            'agg_applicant_education_M',\n            'mean_dpdmax_139P',\n            'lastcancelreason_561M',\n\n            'mode1_relationshiptoclient_415T',\n            'min_dateofcredstart_739D',\n            'agg_max_amount_A',\n            'datefirstoffer_1144D',\n            'agg_currtotaldebt_A',\n\n            'mode1_incometype_1044T',\n            'mode1_numberofoverdueinstlmax_1151L',\n            'mode1_additional_param_numberofoverdueinstlmax_1151L',\n            'agg_mean_overdueamountmax_35398A',\n            'above_0.0_share_pmts_dpd_303P',\n    \n            'monthsannuity_845L',\n            'lastdelinqdate_224D',\n            'isbidproduct_1095L',\n            'maxdpdtolerance_374P',\n            'mean_residualamount_856A',\n            'std_overdueamount_659A',\n            'agg_maxdbddpd_P',\n            'mode1_numberofoverdueinstlmax_1039L',\n            'agg_numberofcontrsvalue_L',\n            'mean_maxdpdtolerance_577P',\n            'mean_totalamount_6A',\n            'agg_numinstunpaidmax_L',\n            'applicationscnt_867L',\n\n            'sub_numincomingpmts_3546848L_numinstlswithdpd10_728L',\n            'sub_numinstlswithdpd5_4187116L_numinstlswithoutdpd_562L',\n            'div_inittransactionamount_650A_price_1097A',\n            'min_credacc_actualbalance_314A',\n            'add_mode1_numberofoverdueinstlmax_1039L_mode1_numberofoverdueinstlmax_1151L',\n            'sub_agg_currtotaldebt_A_max_outstandingdebt_522A',\n            'sub_maxdbddpdlast1m_3658939P_maxdbddpdtollast12m_3658940P',\n            'agg_totaldebtoverduevalue_A',\n            'min_mainoccupationinc_437A',\n        ]\n    \n    @staticmethod\n    def _get_secondary_features():\n        return [\n            'agg_credit_bureau_queries_L',\n            'maritalst_385M',\n            'responsedate_4527233D',\n            'max_employedfrom_700D',\n            'mode1_education_1138M',\n            'mode1_additional_param_postype_4733339M',\n            'mode1_additional_param_rejectreasonclient_4145042M',\n            'mode1_isdebitcard_527L',\n            'agg_max_num_group1_3_4',\n            'max_instlamount_852A',\n            'max_monthlyinstlamount_332A',\n            'min_totaloutstanddebtvalue_39A',\n            'mean_totalamount_996A',\n            'agg_std_overdueamountmax_35398A',\n            'applicant_max_purposeofcred_426M',\n            'mode1_collater_valueofguarantee_1124L',\n\n            'agg_mean_amount_A',\n            'agg_overdueamountmaxdateyear_T',\n            'agg_applicant_overdueamountmaxdateyear_T',\n            'agg_mode1_additional_param_13_545_591_426_M',\n            'cntincpaycont9m_3716944L',\n            'lastrejectreason_759M',\n            'posfpd30lastmonth_3976960P',\n            'max_credamount_590A',\n            'mean_credamount_590A',\n            'max_overdueamountmax2date_1142D',\n            'applicant_max_dateofcredend_289D',\n            'applicant_max_overdueamountmax2date_1002D',\n            'mode1_additional_param_purposeofcred_874M',\n            'mode1_sex_738L',\n            'max_amount_416A',\n            'mean_amount_416A',\n            'max_pmts_overdue_1152A',\n            \n            'equalitydataagreement_891L',\n            'lastapprcredamount_781A',\n            'lastrejectcredamount_222A',\n            'maxdpdfrom6mto36m_3546853P',\n            'mode1_financialinstitution_382M',  # check\n\n            # 'sub_agg_totaldebtoverdue_A_max_overdueamount_659A',\n            'sub_agg_totaldebtoverdue_A_agg_max_overdueamountmax_14155A',\n            'sub_mean_totalamount_996A_mean_totalamount_6A',\n            'div_mean_totaloutstanddebtvalue_39A_maininc_215A',\n            'div_maxdpdlast12m_727P_maininc_215A',\n            'div_agg_currtotaldebt_A_price_1097A',\n            'div_mean_monthlyinstlamount_332A_maininc_215A',\n            'agg_numinstpaid_L',\n            'agg_dtlastpmtallstes_D',\n            'agg_creationdate_firstnonzeroinstldate_D',\n            'agg_max_overdueamountmax_35398A',\n            'thirdquarter_1082L',\n            'avgdbddpdlast3m_4187120P',\n            'credamount_770A',\n            'credtype_322L',\n            'daysoverduetolerancedd_3976961L',\n            'inittransactioncode_186L',\n            'lastrejectcommoditycat_161M',\n            'maxannuity_159A',\n            'numinstls_657L',\n            'numinstpaidlate1d_3546852L',\n            'sellerplacescnt_216L',\n            'max_credacc_maxhisbal_375A',\n            'max_credacc_minhisbal_90A',\n            'min_credacc_maxhisbal_375A',\n            'min_credacc_minhisbal_90A',\n            'mean_credacc_minhisbal_90A',\n            'mean_currdebt_94A',\n            'mean_mainoccupationinc_437A',\n            'mean_revolvingaccount_394A',\n            'applicant_max_credamount_590A',\n            'applicant_max_currdebt_94A',\n            'applicant_max_downpmt_134A',\n            'mode1_credacc_status_367L',\n            'mode1_credtype_587L',\n            'mode1_familystate_726L',\n            'max_credlmt_935A',\n            'max_outstandingamount_362A',\n            'min_totalamount_6A',\n            'mean_overdueamount_659A',\n            'std_instlamount_768A',\n            'std_monthlyinstlamount_674A',\n            'applicant_max_totalamount_996A',\n            'max_dateofcredend_289D',\n            'max_dateofcredstart_739D',\n            'min_numberofoverdueinstlmaxdat_641D',\n            'mode1_nominalrate_281L',\n            'mode1_nominalrate_498L',\n            'mode1_numberofinstls_229L',\n            'mode1_periodicityofpmts_1102L',\n            'applicant_max_numberofoverdueinstlmax_1039L',\n            'mode1_collater_valueofguarantee_876L',\n            \n            'mean_outstandingdebt_522A',\n            'agg_mean_overdueamountmax_14155A',\n            'agg_std_amount_A',\n            'agg_firstnonzeroinstldate_D',\n            'applicant_max_familystate_726L',\n            'applicant_max_dateofcredstart_739D',\n            'min_overdueamountmax2date_1002D',\n            'maxdbddpdtollast12m_3658940P',\n            'maxdbddpdlast1m_3658939P',\n            'agg_maxdpdlast_P',\n            'div_annuity_780A_maininc_215A',\n            'div_agg_sumoutstandtotal_A_price_1097A',\n            'sub_mean_totaldebtoverduevalue_178A_mean_pmts_overdue_1140A',\n\n            'add_agg_currtotaldebt_A_agg_sumoutstandtotal_A',\n            'add_mean_pmts_overdue_1140A_mean_pmts_overdue_1152A',\n            'sub_annuity_780A_applicant_max_annuity_853A',\n            'sub_agg_avgdbddpdlast24m_P_avgdbddpdlast3m_4187120P',\n            'sub_mean_monthlyinstlamount_332A_mean_monthlyinstlamount_674A',\n            'div_avglnamtstart24m_4525187A_maininc_215A',\n            'div_maxpmtlast3m_4525190A_maininc_215A',\n            'agg_min_dtlastpmt_D',\n            'agg_mindpdlast24m_P',\n            'agg_numinstregularpaid_L',\n            'agg_pctinstlsallpaidlat_L',\n            'agg_activateddate_dateactivated_D',\n            'agg_sumoutstandtotal_A',\n            'agg_dpdmax_139_pmts_dpd_1073_P',\n            'agg_dpdmax_757_pmts_dpd_303_P',\n            'agg_approvaldate_firstnonzeroinstldate_D',\n            'agg_min_amount_A',\n            'agg_applicant_credlmt_totalamount_A',\n            'agg_mode1_additional_param_empladdr_district_zipcode_M',\n            'agg_min_openingdate_D',\n            'agg_max_overdueamountmax_14155A',\n            'agg_min_overdueamountmax_14155A',\n            'agg_min_overdueamountmax_35398A',\n            'agg_applicant_max_overdueamountmax_14155A',\n            'avglnamtstart24m_4525187A',\n            'avgmaxdpdlast9m_3716943P',\n            'avgpmtlast12m_4525200A',\n            'cntpmts24_3658933L',\n            'datelastinstal40dpd_247D',\n            'disbursedcredamount_1113A',\n            'homephncnt_628L',\n            'inittransactionamount_650A',\n            'lastapprcommoditycat_1041M',\n            'lastrejectreasonclient_4145040M',\n            'lastst_736L',\n            'maxdebt4_972A',\n            'maxdpdlast24m_143P',\n            'maxdpdlast9m_1059P',\n            'maxinstallast24m_3658928A',\n            'maxlnamtstart6m_4525199A',\n            'numcontrs3months_479L',\n            'numincomingpmts_3546848L',\n            'numnotactivated_1143L',\n            'numrejects9m_859L',\n            'pctinstlsallpaidearl3d_427L',\n            'totalsettled_863A',\n            'totinstallast1m_4525188A',\n            'max_maxdpdtolerance_577P',\n            'max_annuity_853A',\n            'max_mainoccupationinc_437A',\n            'max_outstandingdebt_522A',\n            'mean_annuity_853A',\n            'std_credamount_590A',\n            'applicant_max_annuity_853A',\n            'min_employedfrom_700D',\n            'applicant_max_employedfrom_700D',\n            'mode1_childnum_21L',\n            'applicant_max_credtype_587L',\n            'applicant_max_inittransactioncode_279L',\n            'mean_dpdmax_757P',\n            'applicant_max_dpdmax_757P',\n            'max_credlmt_230A',\n            'max_instlamount_768A',\n            'max_monthlyinstlamount_674A',\n            'max_residualamount_488A',\n            'max_residualamount_856A',\n            'max_totalamount_6A',\n            'max_totalamount_996A',\n            'min_credlmt_230A',\n            'min_instlamount_852A',\n            'min_monthlyinstlamount_332A',\n            'min_monthlyinstlamount_674A',\n            'min_totaldebtoverduevalue_178A',\n            'mean_credlmt_230A',\n            'mean_outstandingamount_362A',\n            'mean_residualamount_488A',\n            'mean_totaldebtoverduevalue_178A',\n            'std_instlamount_852A',\n            'std_monthlyinstlamount_332A',\n            'std_residualamount_488A',\n            'std_residualamount_856A',\n            'std_totalamount_6A',\n            'applicant_max_monthlyinstlamount_332A',\n            'max_dateofcredstart_181D',\n            'max_dateofrealrepmt_138D',\n            'max_lastupdate_388D',\n            'max_numberofoverdueinstlmaxdat_641D',\n            'max_overdueamountmax2date_1002D',\n            'min_dateofcredend_289D',\n            'min_dateofcredstart_181D',\n            'min_dateofrealrepmt_138D',\n            'applicant_max_dateofcredend_353D',\n            'applicant_max_dateofrealrepmt_138D',\n            'applicant_max_lastupdate_388D',\n            'applicant_max_refreshdate_3813885D',\n            'mode1_numberofinstls_320L',\n            'mode1_prolongationcount_1120L',\n            'applicant_max_numberofinstls_320L',\n            'applicant_max_numberofoverdueinstlmax_1151L',\n            'mean_pmts_overdue_1152A',\n            'maxdpdinstldate_3546855D',\n            'maxdpdlast3m_392P',\n            'mode1_numberofoverdueinstls_725L',\n            'agg_numinstnotpaid_L',\n\n            'mode1_rejectreason_755M',\n            'mode1_financialinstitution_591M',\n            'mode1_familystate_447L',\n            'mode1_education_927M',\n            'datelastunpaid_3546854D',\n            'mode1_cancelreason_3545846M',\n        ]","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:23:31.619733Z","iopub.execute_input":"2024-05-14T20:23:31.620012Z","iopub.status.idle":"2024-05-14T20:23:31.704322Z","shell.execute_reply.started":"2024-05-14T20:23:31.619990Z","shell.execute_reply":"2024-05-14T20:23:31.703443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Construct the training dataset","metadata":{}},{"cell_type":"code","source":"%%time\n\npreprocessor = Preprocessor()\npreprocessor.fit(df_train)\n\nX, y = preprocessor.transform(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:23:31.705342Z","iopub.execute_input":"2024-05-14T20:23:31.705609Z","iopub.status.idle":"2024-05-14T20:31:29.569276Z","shell.execute_reply.started":"2024-05-14T20:23:31.705587Z","shell.execute_reply":"2024-05-14T20:31:29.568276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert df_train.shape[0] == X.shape[0]\nassert y.shape == (X.shape[0],)\nassert (X.index == y.index).all()\n\nfor i in range(1, X.shape[0]):\n    assert X['WEEK_NUM'].iloc[i - 1] <= X['WEEK_NUM'].iloc[i]\n\nfor col in X.columns:\n    assert X[col].isna().sum() <= 0.95 * X.shape[0]\n    assert X[col].unique().shape[0] >= 2\n    assert X[col].dtype.name != 'object'\n\n    if X[col].dtype.name == 'category':\n        assert X[col].isna().sum() == 0","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:31:29.570777Z","iopub.execute_input":"2024-05-14T20:31:29.571172Z","iopub.status.idle":"2024-05-14T20:32:20.309210Z","shell.execute_reply.started":"2024-05-14T20:31:29.571134Z","shell.execute_reply":"2024-05-14T20:32:20.308420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, X.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:32:20.310290Z","iopub.execute_input":"2024-05-14T20:32:20.310644Z","iopub.status.idle":"2024-05-14T20:32:20.317327Z","shell.execute_reply.started":"2024-05-14T20:32:20.310614Z","shell.execute_reply":"2024-05-14T20:32:20.316428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:32:20.318429Z","iopub.execute_input":"2024-05-14T20:32:20.318701Z","iopub.status.idle":"2024-05-14T20:32:20.464702Z","shell.execute_reply.started":"2024-05-14T20:32:20.318679Z","shell.execute_reply":"2024-05-14T20:32:20.463737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model","metadata":{}},{"cell_type":"code","source":"# m1 = VotingModel._fit_lgb(\n#     X.drop(columns=['WEEK_NUM']).iloc[:2000], y.iloc[:2000],\n#     X.drop(columns=['WEEK_NUM']).iloc[2000:5000], y.iloc[2000:5000]\n# )","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:32:20.465973Z","iopub.execute_input":"2024-05-14T20:32:20.466348Z","iopub.status.idle":"2024-05-14T20:32:20.474455Z","shell.execute_reply.started":"2024-05-14T20:32:20.466315Z","shell.execute_reply":"2024-05-14T20:32:20.473575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# m2 = VotingModel._fit_cat(\n#     X.drop(columns=['WEEK_NUM']).iloc[:2000], y.iloc[:2000],\n#     X.drop(columns=['WEEK_NUM']).iloc[2000:5000], y.iloc[2000:5000]\n# )","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:32:20.475717Z","iopub.execute_input":"2024-05-14T20:32:20.475987Z","iopub.status.idle":"2024-05-14T20:32:20.483860Z","shell.execute_reply.started":"2024-05-14T20:32:20.475966Z","shell.execute_reply":"2024-05-14T20:32:20.482988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel = VotingModel(n_splits=7)\n# model.fit(X.iloc[:200000], y.iloc[:200000])\nmodel.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:32:20.485042Z","iopub.execute_input":"2024-05-14T20:32:20.485339Z","iopub.status.idle":"2024-05-14T20:39:40.081832Z","shell.execute_reply.started":"2024-05-14T20:32:20.485316Z","shell.execute_reply":"2024-05-14T20:39:40.080845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(model.n_splits):\n#     model.estimators[i].booster_.save_model(\n#         f'/kaggle/working/lgb_model_{i}.txt'\n#         # num_iteration=model.estimators[i].best_iteration\n#     )\n\nfor i in range(model.n_splits):\n    model.estimators[i].save_model(\n        f'/kaggle/working/cat_model_{i}'\n    )\n\n# for i, estimator in enumerate(model.estimators):\n#     path_to_out_file = f'/kaggle/working/estimator_{i}.pickle'\n#     with open(path_to_out_file, 'wb') as fout:\n#         pickle.dump(model, fout)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:40.083112Z","iopub.execute_input":"2024-05-14T20:39:40.083464Z","iopub.status.idle":"2024-05-14T20:39:40.089001Z","shell.execute_reply.started":"2024-05-14T20:39:40.083432Z","shell.execute_reply":"2024-05-14T20:39:40.087945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(16, 6))\n\nfor i, eval_result in enumerate(model.eval_results):\n    # plt.plot(eval_result['train'], label=f'train_{i}')\n    plt.plot(eval_result['valid_metric'], label=f'valid_metric_{i}')\n\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:40.089967Z","iopub.execute_input":"2024-05-14T20:39:40.090218Z","iopub.status.idle":"2024-05-14T20:39:40.301452Z","shell.execute_reply.started":"2024-05-14T20:39:40.090196Z","shell.execute_reply":"2024-05-14T20:39:40.300501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi_df = pd.DataFrame(\n    index=model.feature_names,\n    data=np.array(model.feature_importances).T,\n    columns=[f'estimator_{i}' for i in range(len(model.feature_importances))]\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:40.302866Z","iopub.execute_input":"2024-05-14T20:39:40.303208Z","iopub.status.idle":"2024-05-14T20:39:41.414360Z","shell.execute_reply.started":"2024-05-14T20:39:40.303184Z","shell.execute_reply":"2024-05-14T20:39:41.411362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi_mean = fi_df.mean(axis=1).sort_values(ascending=False)\nfi_std = fi_df.std(axis=1).loc[fi_mean.index]\n\nfi_mean.plot(\n    kind='barh',\n    xerr=fi_std,\n    figsize=(16, fi_mean.shape[0] / 2)\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.415282Z","iopub.status.idle":"2024-05-14T20:39:41.415667Z","shell.execute_reply.started":"2024-05-14T20:39:41.415461Z","shell.execute_reply":"2024-05-14T20:39:41.415476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cv_result in model.cv_results:\n    print(cv_result['valid_roc_auc_score'], cv_result['valid_stability_score'])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.417178Z","iopub.status.idle":"2024-05-14T20:39:41.417495Z","shell.execute_reply.started":"2024-05-14T20:39:41.417343Z","shell.execute_reply":"2024-05-14T20:39:41.417356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stability_scores = np.array([\n    fold_result['valid_stability_score'] for fold_result in model.cv_results\n])\nprint(np.mean(stability_scores), np.std(stability_scores))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.418811Z","iopub.status.idle":"2024-05-14T20:39:41.419144Z","shell.execute_reply.started":"2024-05-14T20:39:41.418978Z","shell.execute_reply":"2024-05-14T20:39:41.418992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor cv_result in model.cv_results:\n    y_true = cv_result['y_valid']\n    y_pred = cv_result['y_pred_valid']\n    neg_class_scores = y_pred[y_true <= 0.5]\n    pos_class_scores = y_pred[y_true > 0.5]\n    # diff_scores = np.subtract.outer(pos_class_scores, neg_class_scores)\n    print('==============================================')\n    print(y_pred.mean(), y_pred.std())\n    # print(np.mean(diff_scores), np.std(diff_scores))\n    auc1 = roc_auc_score(y_true, y_pred)\n    auc2 = roc_auc_score(\n        y_true,\n        y_pred + np.random.normal(0.0, 0.01, size=y_pred.shape[0])\n    )\n    print('Gini decrease:', 2 * (auc1 - auc2))\n    print('==============================================')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.420099Z","iopub.status.idle":"2024-05-14T20:39:41.420427Z","shell.execute_reply.started":"2024-05-14T20:39:41.420267Z","shell.execute_reply":"2024-05-14T20:39:41.420281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trees_n = 3\n# fig, ax = plt.subplots(nrows=trees_n, figsize=(16, 6 * trees_n), sharex=True)\n# for i in range(trees_n):\n#     lgb.plot_tree(model.estimators[0], tree_index=i, dpi=300, ax=ax[i])","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.421808Z","iopub.status.idle":"2024-05-14T20:39:41.422107Z","shell.execute_reply.started":"2024-05-14T20:39:41.421957Z","shell.execute_reply":"2024-05-14T20:39:41.421970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ndel y\ndel fi_mean\ndel fi_std\ndel fi_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.423254Z","iopub.status.idle":"2024-05-14T20:39:41.423602Z","shell.execute_reply.started":"2024-05-14T20:39:41.423409Z","shell.execute_reply":"2024-05-14T20:39:41.423422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Files","metadata":{}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.424681Z","iopub.status.idle":"2024-05-14T20:39:41.425002Z","shell.execute_reply.started":"2024-05-14T20:39:41.424844Z","shell.execute_reply":"2024-05-14T20:39:41.424857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.426471Z","iopub.status.idle":"2024-05-14T20:39:41.426807Z","shell.execute_reply.started":"2024-05-14T20:39:41.426652Z","shell.execute_reply":"2024-05-14T20:39:41.426665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T20:39:41.428662Z","iopub.status.idle":"2024-05-14T20:39:41.428980Z","shell.execute_reply.started":"2024-05-14T20:39:41.428831Z","shell.execute_reply":"2024-05-14T20:39:41.428844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.select([col for col in train_cols if col != \"target\"])\nassert len(df_test.shape) == 2\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test, _ = to_pandas(df_test, cat_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test, _ = preprocessor.transform(df_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# t = df_test['min_refreshdate_3813885D'].rank(ascending=False)\n# t = t.fillna(t.max())\n# assert t.isna().sum() == 0\n# assert (t >= 0).all()\n\n# scaled_t = (t - t.min()) / (t.max() - t.min())\n# assert (0.0 <= scaled_t).all() and (scaled_t <= 1.0).all()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.shape, X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict_proba(X_test)[:, 1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.random.seed(42)\n# y_pred += np.random.normal(\n#     loc=np.zeros(y_pred.shape[0]),\n#     scale=(0.01 * (1.0 - scaled_t)).to_numpy(),\n#     size=y_pred.shape[0]\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}