# %% [code] # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:19.426949Z","iopub.execute_input":"2024-02-07T21:14:19.427573Z","iopub.status.idle":"2024-02-07T21:14:25.033388Z","shell.execute_reply.started":"2024-02-07T21:14:19.427542Z","shell.execute_reply":"2024-02-07T21:14:25.03244Z"}} import os import gc from glob import glob from pathlib import Path from datetime import datetime import numpy as np import pandas as pd import polars as pl import matplotlib.pyplot as plt import seaborn as sns from sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold from sklearn.base import BaseEstimator, RegressorMixin import joblib import lightgbm as lgb import warnings warnings.simplefilter(action='ignore', category=FutureWarning) # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.03567Z","iopub.execute_input":"2024-02-07T21:14:25.036613Z","iopub.status.idle":"2024-02-07T21:14:25.044754Z","shell.execute_reply.started":"2024-02-07T21:14:25.036554Z","shell.execute_reply":"2024-02-07T21:14:25.043557Z"}} def set_table_dtypes(df): for col in df.columns: if col[-1] in ("P", "A"): df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col)) if col[-1] in ("M"): df = df.with_columns(pl.col(col).cast(pl.String).alias(col)) return df # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.046401Z","iopub.execute_input":"2024-02-07T21:14:25.046997Z","iopub.status.idle":"2024-02-07T21:14:25.058163Z","shell.execute_reply.started":"2024-02-07T21:14:25.046956Z","shell.execute_reply":"2024-02-07T21:14:25.056832Z"}} def handle_dates(df): for col in df.columns: if col[-1] in ("D"): df = df.with_columns(pl.col(col).cast(pl.Date).alias(col)) df = df.with_columns(pl.col(col) - pl.col("date_decision")) df = df.with_columns(pl.col(col).dt.total_days()) return df # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.061693Z","iopub.execute_input":"2024-02-07T21:14:25.062196Z","iopub.status.idle":"2024-02-07T21:14:25.074072Z","shell.execute_reply.started":"2024-02-07T21:14:25.062153Z","shell.execute_reply":"2024-02-07T21:14:25.073108Z"}} def filter_cols(df): # Drop if null count of column higher than 80% for col in df.columns: if col in ["case_id", "WEEK_NUM"]: continue isnull = df[col].is_null().mean() if isnull > 0.8: df = df.drop(col) # Drop if number of unique values of column is not between 2-100 for col in df.columns[1:]: if col in ["case_id", "WEEK_NUM"]: continue if df[col].dtype != pl.String: continue freq = df[col].n_unique() if (freq == 1) | (freq > 100): df = df.drop(col) return df # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.075878Z","iopub.execute_input":"2024-02-07T21:14:25.076755Z","iopub.status.idle":"2024-02-07T21:14:25.084091Z","shell.execute_reply.started":"2024-02-07T21:14:25.076698Z","shell.execute_reply":"2024-02-07T21:14:25.083043Z"}} def read_file(path): df = pl.read_parquet(path) df = df.pipe(set_table_dtypes) return df # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.086339Z","iopub.execute_input":"2024-02-07T21:14:25.087106Z","iopub.status.idle":"2024-02-07T21:14:25.098482Z","shell.execute_reply.started":"2024-02-07T21:14:25.087073Z","shell.execute_reply":"2024-02-07T21:14:25.09681Z"}} def feature_eng(df_base, df_person_1, df_static, df_static_cb, df_credit_bureau_b_2): df_base = ( df_base .with_columns( date_decision = pl.col("date_decision").cast(pl.Date), WEEK_NUM = pl.col("WEEK_NUM").cast(pl.Int32), ) .with_columns( month_decision = pl.col("date_decision").dt.month(), weekday_decision = pl.col("date_decision").dt.weekday(), ) ) df_person_1 = ( df_person_1 .group_by("case_id") .agg( [pl.max(col) for col in df_person_1.columns if col != "case_id"], ) ) df_credit_bureau_b_2 = ( df_credit_bureau_b_2 .group_by("case_id") .agg( [pl.max(col) for col in df_credit_bureau_b_2.columns if col != "case_id"], ) ) df_data = ( df_base .join(df_person_1, how="left", on="case_id", suffix="_p1") .join(df_static, how="left", on="case_id", suffix="_s") .join(df_static_cb, how="left", on="case_id", suffix="_scb") .join(df_credit_bureau_b_2, how="left", on="case_id", suffix="cbb2") ) return df_data # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.099938Z","iopub.execute_input":"2024-02-07T21:14:25.100355Z","iopub.status.idle":"2024-02-07T21:14:25.111811Z","shell.execute_reply.started":"2024-02-07T21:14:25.100324Z","shell.execute_reply":"2024-02-07T21:14:25.110331Z"}} def to_pandas(df_data, cat_cols=None): df_data = df_data.to_pandas() if cat_cols is None: cat_cols = list(df_data.select_dtypes("object").columns) df_data[cat_cols] = df_data[cat_cols].astype("category") return df_data, cat_cols # %% [markdown] # #### Configuration # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.11358Z","iopub.execute_input":"2024-02-07T21:14:25.114844Z","iopub.status.idle":"2024-02-07T21:14:25.125505Z","shell.execute_reply.started":"2024-02-07T21:14:25.114796Z","shell.execute_reply":"2024-02-07T21:14:25.12394Z"}} ROOT = Path("/kaggle/input/home-credit-credit-risk-model-stability") TRAIN_DIR = ROOT / "parquet_files" / "train" TEST_DIR = ROOT / "parquet_files" / "test" BASE_TRAIN_PATH = TRAIN_DIR / "train_base.parquet" BASE_TEST_PATH = TRAIN_DIR / "train_base.parquet" MODEL_PATH = "/kaggle/input/new-credit-model/w_new_best_model.pth"#"/kaggle/input/homecredit-dataset/best_model.pth" LOAD_MODEL = True # %% [markdown] # #### Train Files Read & Feature Engineering # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:25.126967Z","iopub.execute_input":"2024-02-07T21:14:25.127343Z","iopub.status.idle":"2024-02-07T21:14:31.531223Z","shell.execute_reply.started":"2024-02-07T21:14:25.127313Z","shell.execute_reply":"2024-02-07T21:14:31.529946Z"}} df_base = read_file(TRAIN_DIR / "train_base.parquet") df_static_cb = read_file(TRAIN_DIR / "train_static_cb_0.parquet") df_person_1 = read_file(TRAIN_DIR / "train_person_1.parquet") df_credit_bureau_b_2 = read_file(TRAIN_DIR / "train_credit_bureau_b_2.parquet") df_static = pl.concat([ read_file(TRAIN_DIR / "train_static_0_0.parquet"), read_file(TRAIN_DIR / "train_static_0_1.parquet"), ], how="vertical_relaxed") # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:14:31.535071Z","iopub.execute_input":"2024-02-07T21:14:31.535465Z","iopub.status.idle":"2024-02-07T21:15:00.688659Z","shell.execute_reply.started":"2024-02-07T21:14:31.535431Z","shell.execute_reply":"2024-02-07T21:15:00.686822Z"}} df_train = feature_eng(df_base, df_person_1, df_static, df_static_cb, df_credit_bureau_b_2) df_train = df_train.pipe(handle_dates) df_train = df_train.pipe(filter_cols) df_train, cat_cols = to_pandas(df_train) # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:00.690274Z","iopub.execute_input":"2024-02-07T21:15:00.690775Z","iopub.status.idle":"2024-02-07T21:15:00.699014Z","shell.execute_reply.started":"2024-02-07T21:15:00.690729Z","shell.execute_reply":"2024-02-07T21:15:00.697647Z"}} print("base shape:\t\t", df_base.shape) print("person_1 shape:\t\t", df_person_1.shape) print("static shape:\t\t", df_static.shape) print("static_cb shape:\t", df_static_cb.shape) print("credit_bureau_b_2 shape:", df_credit_bureau_b_2.shape) print("train data shape:\t", df_train.shape) # %% [markdown] # #### Test Files Read & Feature Engineering # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:00.700318Z","iopub.execute_input":"2024-02-07T21:15:00.700887Z","iopub.status.idle":"2024-02-07T21:15:01.119074Z","shell.execute_reply.started":"2024-02-07T21:15:00.700855Z","shell.execute_reply":"2024-02-07T21:15:01.117886Z"}} df_base = read_file(TEST_DIR / "test_base.parquet") df_static_cb = read_file(TEST_DIR / "test_static_cb_0.parquet") df_person_1 = read_file(TEST_DIR / "test_person_1.parquet") df_credit_bureau_b_2 = read_file(TEST_DIR / "test_credit_bureau_b_2.parquet") df_static = pl.concat([ read_file(TEST_DIR / "test_static_0_0.parquet"), read_file(TEST_DIR / "test_static_0_1.parquet"), read_file(TEST_DIR / "test_static_0_2.parquet"), ], how="vertical_relaxed") # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:01.120426Z","iopub.execute_input":"2024-02-07T21:15:01.120813Z","iopub.status.idle":"2024-02-07T21:15:01.180421Z","shell.execute_reply.started":"2024-02-07T21:15:01.12078Z","shell.execute_reply":"2024-02-07T21:15:01.179265Z"}} df_test = feature_eng(df_base, df_person_1, df_static, df_static_cb, df_credit_bureau_b_2) df_test = df_test.pipe(handle_dates) df_test = df_test.select(df_train.columns.drop("target")) df_test, _ = to_pandas(df_test, cat_cols) # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:01.182202Z","iopub.execute_input":"2024-02-07T21:15:01.182671Z","iopub.status.idle":"2024-02-07T21:15:01.190993Z","shell.execute_reply.started":"2024-02-07T21:15:01.182624Z","shell.execute_reply":"2024-02-07T21:15:01.189575Z"}} print("base shape:\t\t", df_base.shape) print("person_1 shape:\t\t", df_person_1.shape) print("static shape:\t\t", df_static.shape) print("static_cb shape:\t", df_static_cb.shape) print("credit_bureau_b_2 shape:", df_credit_bureau_b_2.shape) print("test data shape:\t", df_test.shape) # %% [markdown] # #### Memory Cleaning # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:01.192202Z","iopub.execute_input":"2024-02-07T21:15:01.192584Z","iopub.status.idle":"2024-02-07T21:15:01.326172Z","shell.execute_reply.started":"2024-02-07T21:15:01.192539Z","shell.execute_reply":"2024-02-07T21:15:01.324805Z"}} del df_base del df_person_1 del df_static del df_static_cb del df_credit_bureau_b_2 gc.collect() # %% [markdown] # #### EDA # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:01.328002Z","iopub.execute_input":"2024-02-07T21:15:01.329167Z","iopub.status.idle":"2024-02-07T21:15:01.379583Z","shell.execute_reply.started":"2024-02-07T21:15:01.329113Z","shell.execute_reply":"2024-02-07T21:15:01.378409Z"}} print("Train is duplicated:\t", df_train["case_id"].duplicated().any()) print("Train Week Range:\t", (df_train["WEEK_NUM"].min(), df_train["WEEK_NUM"].max())) print() print("Test is duplicated:\t", df_test["case_id"].duplicated().any()) print("Test Week Range:\t", (df_test["WEEK_NUM"].min(), df_test["WEEK_NUM"].max())) # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:01.381057Z","iopub.execute_input":"2024-02-07T21:15:01.381885Z","iopub.status.idle":"2024-02-07T21:15:21.225397Z","shell.execute_reply.started":"2024-02-07T21:15:01.381851Z","shell.execute_reply":"2024-02-07T21:15:21.224126Z"}} sns.lineplot( data=df_train, x="WEEK_NUM", y="target", ) plt.show() # %% [markdown] # #### Training # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:21.227298Z","iopub.execute_input":"2024-02-07T21:15:21.228578Z","iopub.status.idle":"2024-02-07T21:15:21.237612Z","shell.execute_reply.started":"2024-02-07T21:15:21.228489Z","shell.execute_reply":"2024-02-07T21:15:21.236107Z"}} class VotingModel(BaseEstimator, RegressorMixin): def __init__(self, estimators): super().__init__() self.estimators = estimators def fit(self, X, y=None): return self def predict(self, X): y_preds = [estimator.predict(X) for estimator in self.estimators] return np.mean(y_preds, axis=0) def predict_proba(self, X): y_preds = [estimator.predict_proba(X) for estimator in self.estimators] return np.mean(y_preds, axis=0) # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:30.078135Z","iopub.execute_input":"2024-02-07T21:15:30.07851Z","iopub.status.idle":"2024-02-07T21:15:30.090339Z","shell.execute_reply.started":"2024-02-07T21:15:30.07848Z","shell.execute_reply":"2024-02-07T21:15:30.088894Z"}} if not LOAD_MODEL: X = df_train.drop(columns=["target", "case_id", "date_decision", "WEEK_NUM", "MONTH"]) y = df_train["target"] weeks = df_train["WEEK_NUM"] cv = StratifiedGroupKFold(n_splits=5, shuffle=False) params = { "boosting_type": "gbdt", "objective": "binary", "metric": "auc", "max_depth": 8, "learning_rate": 0.05, "n_estimators": 1000, "colsample_bytree": 0.8, "colsample_bynode": 0.8, "verbose": -1, } fitted_models = [] for idx_train, idx_valid in cv.split(X, y, groups=weeks): X_train, y_train = X.iloc[idx_train], y.iloc[idx_train] X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid] print("Valid week range: ", (weeks.iloc[idx_valid].min(), weeks.iloc[idx_valid].max())) model = lgb.LGBMClassifier(**params) model.fit( X_train, y_train, eval_set=[(X_valid, y_valid)], callbacks=[lgb.log_evaluation(50), lgb.early_stopping(50)] ) fitted_models.append(model) model = VotingModel(fitted_models) # %% [markdown] # #### Save & Load Model/kaggle/input/new-credit-model/w_new_best_model.pth # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:31.619181Z","iopub.execute_input":"2024-02-07T21:15:31.61961Z","iopub.status.idle":"2024-02-07T21:15:31.984436Z","shell.execute_reply.started":"2024-02-07T21:15:31.619576Z","shell.execute_reply":"2024-02-07T21:15:31.983125Z"}} if not LOAD_MODEL: joblib.dump(model, "w_new_best_model.pth") else: model = joblib.load(MODEL_PATH) # %% [markdown] # #### Prediction # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:32.731186Z","iopub.execute_input":"2024-02-07T21:15:32.731581Z","iopub.status.idle":"2024-02-07T21:15:32.92016Z","shell.execute_reply.started":"2024-02-07T21:15:32.73155Z","shell.execute_reply":"2024-02-07T21:15:32.918798Z"}} X_test = df_test.drop(columns=["date_decision", "WEEK_NUM", "MONTH"]) X_test = X_test.set_index("case_id") y_pred = pd.Series(model.predict_proba(X_test)[:, 1], index=X_test.index) # %% [markdown] # #### Submission # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:33.420788Z","iopub.execute_input":"2024-02-07T21:15:33.421223Z","iopub.status.idle":"2024-02-07T21:15:33.433211Z","shell.execute_reply.started":"2024-02-07T21:15:33.421193Z","shell.execute_reply":"2024-02-07T21:15:33.431801Z"}} df_subm = pd.read_csv(ROOT / "sample_submission.csv") df_subm = df_subm.set_index("case_id") df_subm["score"] = y_pred # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:33.780338Z","iopub.execute_input":"2024-02-07T21:15:33.780743Z","iopub.status.idle":"2024-02-07T21:15:33.790484Z","shell.execute_reply.started":"2024-02-07T21:15:33.780694Z","shell.execute_reply":"2024-02-07T21:15:33.789154Z"}} print("Check null: ", df_subm["score"].isnull().any()) # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:34.032756Z","iopub.execute_input":"2024-02-07T21:15:34.033226Z","iopub.status.idle":"2024-02-07T21:15:34.053276Z","shell.execute_reply.started":"2024-02-07T21:15:34.033195Z","shell.execute_reply":"2024-02-07T21:15:34.051923Z"}} df_subm.head() # %% [code] {"execution":{"iopub.status.busy":"2024-02-07T21:15:34.497348Z","iopub.execute_input":"2024-02-07T21:15:34.497748Z","iopub.status.idle":"2024-02-07T21:15:34.505377Z","shell.execute_reply.started":"2024-02-07T21:15:34.4977Z","shell.execute_reply":"2024-02-07T21:15:34.504263Z"}} df_subm.to_csv("submission.csv") # %% [code]