{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score \nfrom pathlib import Path\nfrom glob import glob\nimport gc\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\nROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:11.096114Z","iopub.execute_input":"2024-04-22T13:29:11.096783Z","iopub.status.idle":"2024-04-22T13:29:17.632008Z","shell.execute_reply.started":"2024-04-22T13:29:11.096718Z","shell.execute_reply":"2024-04-22T13:29:17.630225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        elif col[-1] in (\"M\",):\n            df = df.with_columns(pl.col(col).cast(pl.String).alias(col))\n        elif col[-1] in (\"D\",):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Int64).alias(col))\n        elif col in [\"date_decision\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df\n\ndef filter_cols(df):\n    for col in df.columns:\n        if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n            isnull = df[col].is_null().mean()\n            if isnull > 0.7:\n                df = df.drop(col)\n\n    for col in df.columns:\n        if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n            freq = df[col].n_unique()\n            if (freq == 1) | (freq > 200):\n                df = df.drop(col)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:19.783810Z","iopub.execute_input":"2024-04-22T13:29:19.784831Z","iopub.status.idle":"2024-04-22T13:29:19.804080Z","shell.execute_reply.started":"2024-04-22T13:29:19.784786Z","shell.execute_reply":"2024-04-22T13:29:19.802254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    \"\"\"\n    Helper class taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"    \n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n    \n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n    \n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n    \n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n    \n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]  # max & replace col name\n        return expr_max\n    \n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:23.093144Z","iopub.execute_input":"2024-04-22T13:29:23.093722Z","iopub.status.idle":"2024-04-22T13:29:23.110800Z","shell.execute_reply.started":"2024-04-22T13:29:23.093679Z","shell.execute_reply":"2024-04-22T13:29:23.108693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:25.774631Z","iopub.execute_input":"2024-04-22T13:29:25.776205Z","iopub.status.idle":"2024-04-22T13:29:25.783350Z","shell.execute_reply.started":"2024-04-22T13:29:25.776159Z","shell.execute_reply":"2024-04-22T13:29:25.781770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(regex_path, depth=None):\n    chunks = []\n    for file in glob(str(regex_path)):\n        df = pl.read_parquet(file)\n        df = df.pipe(set_table_dtypes)\n        if depth in [1,2]:\n            df = df.group_by('case_id').agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:27.470333Z","iopub.execute_input":"2024-04-22T13:29:27.471671Z","iopub.status.idle":"2024-04-22T13:29:27.481773Z","shell.execute_reply.started":"2024-04-22T13:29:27.471552Z","shell.execute_reply":"2024-04-22T13:29:27.479628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    \"\"\"\n    Helper function taken from notebook:\n    https://www.kaggle.com/code/daviddirethucus/home-credit-risk-lightgbm\n    \"\"\"\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    df_base = df_base.select(pl.all().exclude(pl.Date))\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df = df.select(pl.all().exclude(pl.Date))\n        df = df.select(pl.all().exclude(pl.String))\n        df = df.select(pl.all().exclude(pl.Object))\n        df = df.select(pl.all().exclude(pl.Null))\n        df = df.select(pl.all().exclude(pl.Boolean))  \n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n#     df_base = df_base.pipe(handle_dates)\n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:29.764733Z","iopub.execute_input":"2024-04-22T13:29:29.766239Z","iopub.status.idle":"2024-04-22T13:29:29.778511Z","shell.execute_reply.started":"2024-04-22T13:29:29.766176Z","shell.execute_reply":"2024-04-22T13:29:29.776487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}\n\ndata_train","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:29:32.209078Z","iopub.execute_input":"2024-04-22T13:29:32.209590Z","iopub.status.idle":"2024-04-22T13:32:13.526231Z","shell.execute_reply.started":"2024-04-22T13:29:32.209551Z","shell.execute_reply":"2024-04-22T13:32:13.524939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}\ndata_test","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:07.326626Z","iopub.execute_input":"2024-04-22T13:33:07.327561Z","iopub.status.idle":"2024-04-22T13:33:07.761170Z","shell.execute_reply.started":"2024-04-22T13:33:07.327485Z","shell.execute_reply":"2024-04-22T13:33:07.759460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_train)\nprint(\"train data shape:\\t\", df_train.shape)\n# clean memory\n# del data_train\ngc.collect()\ndf_train","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:12.524197Z","iopub.execute_input":"2024-04-22T13:33:12.524715Z","iopub.status.idle":"2024-04-22T13:33:20.052612Z","shell.execute_reply.started":"2024-04-22T13:33:12.524678Z","shell.execute_reply":"2024-04-22T13:33:20.051540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_test)\nprint(\"test data shape:\\t\", df_test.shape)\n# clean memory\n# del data_test\ngc.collect()\ndf_test","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:24.739925Z","iopub.execute_input":"2024-04-22T13:33:24.741445Z","iopub.status.idle":"2024-04-22T13:33:24.956608Z","shell.execute_reply.started":"2024-04-22T13:33:24.741397Z","shell.execute_reply":"2024-04-22T13:33:24.955262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.pipe(filter_cols)\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\n\nprint(\"train data shape: \", df_train.shape)\nprint(\"test data shape: \", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:27.724439Z","iopub.execute_input":"2024-04-22T13:33:27.724911Z","iopub.status.idle":"2024-04-22T13:33:28.686847Z","shell.execute_reply.started":"2024-04-22T13:33:27.724878Z","shell.execute_reply":"2024-04-22T13:33:28.685624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.to_pandas()\ndf_test = df_test.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:32.162038Z","iopub.execute_input":"2024-04-22T13:33:32.162593Z","iopub.status.idle":"2024-04-22T13:33:33.514019Z","shell.execute_reply.started":"2024-04-22T13:33:32.162549Z","shell.execute_reply":"2024-04-22T13:33:33.512950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = df_train.iloc[:, 3:].select_dtypes(include='number').columns\ndf_train[num_cols].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:35.577034Z","iopub.execute_input":"2024-04-22T13:33:35.578724Z","iopub.status.idle":"2024-04-22T13:33:39.980016Z","shell.execute_reply.started":"2024-04-22T13:33:35.578674Z","shell.execute_reply":"2024-04-22T13:33:39.978007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids_train, case_ids_test = train_test_split(df_train['case_id'], train_size=0.8, random_state=2024)\nX_train = df_train.loc[df_train['case_id'].isin(case_ids_train)]\nX_test = df_train.loc[df_train['case_id'].isin(case_ids_test)]\ny_train = df_train.loc[df_train['case_id'].isin(case_ids_train), 'target']\ny_test = df_train.loc[df_train['case_id'].isin(case_ids_test), 'target']\n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:42.312987Z","iopub.execute_input":"2024-04-22T13:33:42.313605Z","iopub.status.idle":"2024-04-22T13:33:44.451974Z","shell.execute_reply.started":"2024-04-22T13:33:42.313561Z","shell.execute_reply":"2024-04-22T13:33:44.450344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_new = X_train.drop(columns=['target'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:33:49.340142Z","iopub.execute_input":"2024-04-22T13:33:49.340617Z","iopub.status.idle":"2024-04-22T13:33:50.154587Z","shell.execute_reply.started":"2024-04-22T13:33:49.340580Z","shell.execute_reply":"2024-04-22T13:33:50.152822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_new = X_test.drop(columns=['target'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:34:35.823608Z","iopub.execute_input":"2024-04-22T13:34:35.824107Z","iopub.status.idle":"2024-04-22T13:34:36.060961Z","shell.execute_reply.started":"2024-04-22T13:34:35.824073Z","shell.execute_reply":"2024-04-22T13:34:36.059345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {X_train_new.shape}\")\n# print(f\"Valid: {X_valid.shape}\")\nprint(f\"Test: {X_test_new.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:34:47.076098Z","iopub.execute_input":"2024-04-22T13:34:47.076665Z","iopub.status.idle":"2024-04-22T13:34:47.085280Z","shell.execute_reply.started":"2024-04-22T13:34:47.076623Z","shell.execute_reply":"2024-04-22T13:34:47.083345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits=5, shuffle=False)\nweeks = X_train['WEEK_NUM']\nensemble_models = {}\nlgb_params = {\n    \"boosting_type\": \"gbdt\",\n    'objective' : 'binary',\n    'metric' : 'auc',\n    \"learning_rate\": 0.05,\n    \"max_depth\": 3,\n    \"feature_fraction\": 0.9,\n    \"bagging_fraction\": 0.8,\n    \"bagging_freq\": 5,\n#         \"n_estimators\": 1500,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"random_state\": 2024\n}\nfitted_models = []\ncv_scores = []\nfor i, (train_index, valid_index) in enumerate(cv.split(X_train_new, y_train, groups=weeks)):\n    train_x, train_y = X_train_new.iloc[train_index], y_train.iloc[train_index]\n    valid_x, valid_y = X_train_new.iloc[valid_index], y_train.iloc[valid_index]\n    print(train_x.shape)\n    model = lgb.LGBMClassifier(**lgb_params)\n    model.fit(\n        train_x, train_y,\n        eval_set = [(valid_x, valid_y)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(100)] )\n    fitted_models.append(model)\n    y_pred_valid = model.predict_proba(valid_x)[:,1]\n    auc_score = roc_auc_score(valid_y, y_pred_valid)\n    cv_scores.append(auc_score)\n     # SAVE MODEL\n    ensemble_models[f'model_{i}'] = model\n    \nprint(\"CV AUC scores: \", cv_scores)\nprint(\"Maximum CV AUC score: \", max(cv_scores))","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:49:08.074616Z","iopub.execute_input":"2024-04-22T13:49:08.075150Z","iopub.status.idle":"2024-04-22T13:58:37.240140Z","shell.execute_reply.started":"2024-04-22T13:49:08.075114Z","shell.execute_reply":"2024-04-22T13:58:37.238400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble_models","metadata":{"execution":{"iopub.status.busy":"2024-04-22T13:46:51.389438Z","iopub.execute_input":"2024-04-22T13:46:51.390285Z","iopub.status.idle":"2024-04-22T13:46:51.407714Z","shell.execute_reply.started":"2024-04-22T13:46:51.390245Z","shell.execute_reply":"2024-04-22T13:46:51.406612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = []\nfor key, model in ensemble_models.items():\n    y = model.predict_proba(df_test)[:, 1]\n    y_pred.append(y)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T14:00:07.784498Z","iopub.execute_input":"2024-04-22T14:00:07.785704Z","iopub.status.idle":"2024-04-22T14:00:07.810319Z","shell.execute_reply.started":"2024-04-22T14:00:07.785658Z","shell.execute_reply":"2024-04-22T14:00:07.808802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\ny_pred_array = np.array(y_pred)\nprint(y_pred_array)\ny_pred_ave = np.mean(y_pred_array, axis=0)\nprint(y_pred_ave)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T14:07:50.983445Z","iopub.execute_input":"2024-04-22T14:07:50.984001Z","iopub.status.idle":"2024-04-22T14:07:50.993459Z","shell.execute_reply.started":"2024-04-22T14:07:50.983963Z","shell.execute_reply":"2024-04-22T14:07:50.991821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = y_pred_ave\ndf_subm.to_csv(\"submission.csv\")\ndf_subm","metadata":{"execution":{"iopub.status.busy":"2024-04-22T14:08:11.747412Z","iopub.execute_input":"2024-04-22T14:08:11.748023Z","iopub.status.idle":"2024-04-22T14:08:11.797373Z","shell.execute_reply.started":"2024-04-22T14:08:11.747976Z","shell.execute_reply":"2024-04-22T14:08:11.795944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T14:08:34.059470Z","iopub.execute_input":"2024-04-22T14:08:34.060023Z","iopub.status.idle":"2024-04-22T14:08:34.069066Z","shell.execute_reply.started":"2024-04-22T14:08:34.059984Z","shell.execute_reply":"2024-04-22T14:08:34.067584Z"},"trusted":true},"execution_count":null,"outputs":[]}]}