# %% [code] # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:00:59.009434Z","iopub.execute_input":"2024-04-18T01:00:59.009676Z","iopub.status.idle":"2024-04-18T01:01:03.892791Z","shell.execute_reply.started":"2024-04-18T01:00:59.009654Z","shell.execute_reply":"2024-04-18T01:01:03.89197Z"}} import joblib from pathlib import Path import gc from glob import glob import numpy as np import pandas as pd import polars as pl from sklearn.base import BaseEstimator, RegressorMixin from sklearn.metrics import roc_auc_score import lightgbm as lgb import warnings warnings.filterwarnings('ignore') ROOT = '/kaggle/input/home-credit-credit-risk-model-stability' # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:16.713297Z","iopub.execute_input":"2024-04-18T01:01:16.7139Z","iopub.status.idle":"2024-04-18T01:01:16.736839Z","shell.execute_reply.started":"2024-04-18T01:01:16.713868Z","shell.execute_reply":"2024-04-18T01:01:16.735965Z"}} class Pipeline: def set_table_dtypes(df): for col in df.columns: if col in ["case_id", "WEEK_NUM", "num_group1", "num_group2"]: df = df.with_columns(pl.col(col).cast(pl.Int64)) elif col in ["date_decision"]: df = df.with_columns(pl.col(col).cast(pl.Date)) elif col[-1] in ("P", "A"): df = df.with_columns(pl.col(col).cast(pl.Float64)) elif col[-1] in ("M",): df = df.with_columns(pl.col(col).cast(pl.String)) elif col[-1] in ("D",): df = df.with_columns(pl.col(col).cast(pl.Date)) return df def handle_dates(df): for col in df.columns: if col[-1] in ("D",): df = df.with_columns(pl.col(col) - pl.col("date_decision")) #!!? df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1 df = df.drop("date_decision", "MONTH") return df def filter_cols(df): for col in df.columns: if col not in ["target", "case_id", "WEEK_NUM"]: isnull = df[col].is_null().mean() if isnull > 0.7: df = df.drop(col) for col in df.columns: if (col not in ["target", "case_id", "WEEK_NUM"]) & (df[col].dtype == pl.String): freq = df[col].n_unique() if (freq == 1) | (freq > 200): df = df.drop(col) return df class Aggregator: # Please add or subtract features yourself, be aware that too many features will take up too much space. def num_expr(df): cols = [col for col in df.columns if col[-1] in ("P", "A")] expr_max = [pl.max(col).alias(f"max_{col}") for col in cols] expr_last = [pl.last(col).alias(f"last_{col}") for col in cols] # expr_first = [pl.first(col).alias(f"first_{col}") for col in cols] expr_mean = [pl.mean(col).alias(f"mean_{col}") for col in cols] expr_median = [pl.median(col).alias(f"median_{col}") for col in cols] expr_var = [pl.var(col).alias(f"var_{col}") for col in cols] return expr_max + expr_last + expr_mean def date_expr(df): cols = [col for col in df.columns if col[-1] in ("D")] expr_max = [pl.max(col).alias(f"max_{col}") for col in cols] # expr_min = [pl.min(col).alias(f"min_{col}") for col in cols] expr_last = [pl.last(col).alias(f"last_{col}") for col in cols] # expr_first = [pl.first(col).alias(f"first_{col}") for col in cols] expr_mean = [pl.mean(col).alias(f"mean_{col}") for col in cols] expr_median = [pl.median(col).alias(f"median_{col}") for col in cols] return expr_max + expr_last + expr_mean def str_expr(df): cols = [col for col in df.columns if col[-1] in ("M",)] expr_max = [pl.max(col).alias(f"max_{col}") for col in cols] # expr_min = [pl.min(col).alias(f"min_{col}") for col in cols] expr_last = [pl.last(col).alias(f"last_{col}") for col in cols] # expr_first = [pl.first(col).alias(f"first_{col}") for col in cols] # expr_count = [pl.count(col).alias(f"count_{col}") for col in cols] return expr_max + expr_last # +expr_count def other_expr(df): cols = [col for col in df.columns if col[-1] in ("T", "L")] expr_max = [pl.max(col).alias(f"max_{col}") for col in cols] # expr_min = [pl.min(col).alias(f"min_{col}") for col in cols] expr_last = [pl.last(col).alias(f"last_{col}") for col in cols] # expr_first = [pl.first(col).alias(f"first_{col}") for col in cols] return expr_max + expr_last def count_expr(df): cols = [col for col in df.columns if "num_group" in col] expr_max = [pl.max(col).alias(f"max_{col}") for col in cols] # expr_min = [pl.min(col).alias(f"min_{col}") for col in cols] expr_last = [pl.last(col).alias(f"last_{col}") for col in cols] # expr_first = [pl.first(col).alias(f"first_{col}") for col in cols] return expr_max + expr_last def get_exprs(df): exprs = Aggregator.num_expr(df) + \ Aggregator.date_expr(df) + \ Aggregator.str_expr(df) + \ Aggregator.other_expr(df) + \ Aggregator.count_expr(df) return exprs # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:17.511588Z","iopub.execute_input":"2024-04-18T01:01:17.511938Z","iopub.status.idle":"2024-04-18T01:01:17.531658Z","shell.execute_reply.started":"2024-04-18T01:01:17.5119Z","shell.execute_reply":"2024-04-18T01:01:17.530823Z"}} def read_file(path, depth=None): df = pl.read_parquet(path) df = df.pipe(Pipeline.set_table_dtypes) if depth in [1,2]: df = df.group_by("case_id").agg(Aggregator.get_exprs(df)) return df def read_files(regex_path, depth=None): chunks = [] for path in glob(str(regex_path)): df = pl.read_parquet(path) df = df.pipe(Pipeline.set_table_dtypes) if depth in [1, 2]: df = df.group_by("case_id").agg(Aggregator.get_exprs(df)) chunks.append(df) df = pl.concat(chunks, how="vertical_relaxed") df = df.unique(subset=["case_id"]) return df def feature_eng(df_base, depth_0, depth_1, depth_2): df_base = ( df_base .with_columns( month_decision = pl.col("date_decision").dt.month(), weekday_decision = pl.col("date_decision").dt.weekday(), ) ) for i, df in enumerate(depth_0 + depth_1 + depth_2): df_base = df_base.join(df, how="left", on="case_id", suffix=f"_{i}") df_base = df_base.pipe(Pipeline.handle_dates) return df_base def to_pandas(df_data, cat_cols=None): df_data = df_data.to_pandas() if cat_cols is None: cat_cols = list(df_data.select_dtypes("object").columns) df_data[cat_cols] = df_data[cat_cols].astype("category") return df_data, cat_cols def reduce_mem_usage(df): """ iterate through all the columns of a dataframe and modify the data type to reduce memory usage. """ start_mem = df.memory_usage().sum() / 1024**2 print('Memory usage of dataframe is {:.2f} MB'.format(start_mem)) for col in df.columns: col_type = df[col].dtype if str(col_type)=="category": continue if col_type != object: c_min = df[col].min() c_max = df[col].max() if str(col_type)[:3] == 'int': if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max: df[col] = df[col].astype(np.int8) elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max: df[col] = df[col].astype(np.int16) elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max: df[col] = df[col].astype(np.int32) elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max: df[col] = df[col].astype(np.int64) else: if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max: df[col] = df[col].astype(np.float16) elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max: df[col] = df[col].astype(np.float32) else: df[col] = df[col].astype(np.float64) else: continue end_mem = df.memory_usage().sum() / 1024**2 print('Memory usage after optimization is: {:.2f} MB'.format(end_mem)) print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem)) return df # %% [markdown] # ## Load Models # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:18.449954Z","iopub.execute_input":"2024-04-18T01:01:18.450293Z","iopub.status.idle":"2024-04-18T01:01:19.164628Z","shell.execute_reply.started":"2024-04-18T01:01:18.450267Z","shell.execute_reply":"2024-04-18T01:01:19.163728Z"}} lgb_notebook_info = joblib.load('/kaggle/input/homecredit-models-public/other/lgb/1/notebook_info.joblib') print(f"- [lgb] notebook_start_time: {lgb_notebook_info['notebook_start_time']}") print(f"- [lgb] description: {lgb_notebook_info['description']}") cols = lgb_notebook_info['cols'] cat_cols = lgb_notebook_info['cat_cols'] print(f"- [lgb] len(cols): {len(cols)}") print(f"- [lgb] len(cat_cols): {len(cat_cols)}") lgb_models = joblib.load('/kaggle/input/homecredit-models-public/other/lgb/1/lgb_models.joblib') lgb_models # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:19.166091Z","iopub.execute_input":"2024-04-18T01:01:19.166349Z","iopub.status.idle":"2024-04-18T01:01:23.917777Z","shell.execute_reply.started":"2024-04-18T01:01:19.166327Z","shell.execute_reply":"2024-04-18T01:01:23.916872Z"}} cat_notebook_info = joblib.load('/kaggle/input/homecredit-models-public/other/cat/1/notebook_info.joblib') print(f"- [cat] notebook_start_time: {cat_notebook_info['notebook_start_time']}") print(f"- [cat] description: {cat_notebook_info['description']}") cat_models = joblib.load('/kaggle/input/homecredit-models-public/other/cat/1/cat_models.joblib') cat_models # %% [markdown] # ## Prepare df_test # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:23.919215Z","iopub.execute_input":"2024-04-18T01:01:23.919502Z","iopub.status.idle":"2024-04-18T01:01:24.359795Z","shell.execute_reply.started":"2024-04-18T01:01:23.919477Z","shell.execute_reply":"2024-04-18T01:01:24.3589Z"}} ROOT = Path("/kaggle/input/home-credit-credit-risk-model-stability") TEST_DIR = ROOT / "parquet_files" / "test" data_store = { "df_base": read_file(TEST_DIR / "test_base.parquet"), "depth_0": [ read_file(TEST_DIR / "test_static_cb_0.parquet"), read_files(TEST_DIR / "test_static_0_*.parquet"), ], "depth_1": [ read_files(TEST_DIR / "test_applprev_1_*.parquet", 1), read_file(TEST_DIR / "test_tax_registry_a_1.parquet", 1), read_file(TEST_DIR / "test_tax_registry_b_1.parquet", 1), read_file(TEST_DIR / "test_tax_registry_c_1.parquet", 1), read_files(TEST_DIR / "test_credit_bureau_a_1_*.parquet", 1), read_file(TEST_DIR / "test_credit_bureau_b_1.parquet", 1), read_file(TEST_DIR / "test_other_1.parquet", 1), read_file(TEST_DIR / "test_person_1.parquet", 1), read_file(TEST_DIR / "test_deposit_1.parquet", 1), read_file(TEST_DIR / "test_debitcard_1.parquet", 1), ], "depth_2": [ read_file(TEST_DIR / "test_credit_bureau_b_2.parquet", 2), read_files(TEST_DIR / "test_credit_bureau_a_2_*.parquet", 2), read_file(TEST_DIR / "test_applprev_2.parquet", 2), read_file(TEST_DIR / "test_person_2.parquet", 2) ] } # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:24.373534Z","iopub.execute_input":"2024-04-18T01:01:24.373815Z","iopub.status.idle":"2024-04-18T01:01:24.885224Z","shell.execute_reply.started":"2024-04-18T01:01:24.373792Z","shell.execute_reply":"2024-04-18T01:01:24.884375Z"}} df_test = feature_eng(**data_store) print("test data shape:\t", df_test.shape) del data_store gc.collect() df_test = df_test.select(['case_id'] + cols) df_test, cat_cols = to_pandas(df_test, cat_cols) df_test = reduce_mem_usage(df_test) df_test = df_test.set_index('case_id') print("test data shape:\t", df_test.shape) gc.collect() # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:24.956415Z","iopub.execute_input":"2024-04-18T01:01:24.95721Z","iopub.status.idle":"2024-04-18T01:01:24.99538Z","shell.execute_reply.started":"2024-04-18T01:01:24.957183Z","shell.execute_reply":"2024-04-18T01:01:24.994563Z"}} df_test # %% [markdown] # ## Voting Model # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:28.188831Z","iopub.execute_input":"2024-04-18T01:01:28.189678Z","iopub.status.idle":"2024-04-18T01:01:28.196821Z","shell.execute_reply.started":"2024-04-18T01:01:28.189646Z","shell.execute_reply":"2024-04-18T01:01:28.195974Z"}} class VotingModel(BaseEstimator, RegressorMixin): def __init__(self, estimators): super().__init__() self.estimators = estimators def fit(self, X, y=None): return self def predict(self, X): y_preds = [estimator.predict(X) for estimator in self.estimators] return np.mean(y_preds, axis=0) def predict_proba(self, X): # lgb y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]] # cat X[cat_cols] = X[cat_cols].astype(str) y_preds += [estimator.predict_proba(X) for estimator in self.estimators[-5:]] return np.mean(y_preds, axis=0) # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:28.917023Z","iopub.execute_input":"2024-04-18T01:01:28.917361Z","iopub.status.idle":"2024-04-18T01:01:28.923117Z","shell.execute_reply.started":"2024-04-18T01:01:28.917336Z","shell.execute_reply":"2024-04-18T01:01:28.922171Z"}} model = VotingModel(lgb_models + cat_models) len(model.estimators) # %% [code] {"execution":{"iopub.status.busy":"2024-04-18T01:01:30.256116Z","iopub.execute_input":"2024-04-18T01:01:30.256483Z","iopub.status.idle":"2024-04-18T01:01:30.799928Z","shell.execute_reply.started":"2024-04-18T01:01:30.256455Z","shell.execute_reply":"2024-04-18T01:01:30.799069Z"}} y_pred = pd.Series(model.predict_proba(df_test)[:, 1], index=df_test.index) df_subm = pd.read_csv(ROOT / "sample_submission.csv") df_subm = df_subm.set_index("case_id") df_subm["score"] = y_pred df_subm.to_csv("sub.csv") df_subm