{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introduction\n\nIn this notebook, we will be building a basline model for the Home Credit Kaggle competition. The goal of this competition is to predict the probability of a client defaulting on a loan. Since the dataset is quite large, we will be using Polars to load and process the data. \n\n\n**CHANGELOG**:\n\nv4 -> v5:\nRe-running on modified dataset.\n\nv3 -> v4:\n1. Added min and mean as aggregate statistics in addition to max for tables with depth >= 1\n2. Added a preliminary EDA section. \n","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nfrom typing import List\n\nplt.style.use('ggplot')\nplt.rcParams.update({'figure.dpi':150})","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-11T02:50:44.033558Z","iopub.execute_input":"2024-05-11T02:50:44.033881Z","iopub.status.idle":"2024-05-11T02:50:48.018506Z","shell.execute_reply.started":"2024-05-11T02:50:44.033855Z","shell.execute_reply":"2024-05-11T02:50:48.016860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the data\n\n\n1. We will be using the parquet files provided by the competition. The parquet files are stored in the `parquet_files` directory. The `train` and `test` directories contain the training and testing data, respectively. \n2. We use the Lazy API to read the parquet files using `scan_parquet`. This allows for efficient reading of the data and also allows us to perform operations on the data without loading it into memory until necessary. ","metadata":{}},{"cell_type":"code","source":"TRAIN_DIR = Path('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/')\nTEST_DIR = Path('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:50:51.922953Z","iopub.execute_input":"2024-05-11T02:50:51.923306Z","iopub.status.idle":"2024-05-11T02:50:51.928628Z","shell.execute_reply.started":"2024-05-11T02:50:51.923279Z","shell.execute_reply":"2024-05-11T02:50:51.926932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_dtypes(df: pl.LazyFrame) -> pl.LazyFrame:\n    for column in df.columns:\n        if column in {'case_id', 'WEEK_NUM', 'num_group1', 'num_group2'}:\n            df = df.with_columns(pl.col(column).cast(pl.Int64))\n        elif column == 'date_decision' or column[-1] == 'D':\n            df = df.with_columns(pl.col(column).cast(pl.Date))\n        elif column[-1] in {'A', 'P'}:\n            df = df.with_columns(pl.col(column).cast(pl.Float64))\n        elif column[-1] == 'M':\n            df = df.with_columns(pl.col(column).cast(pl.String))\n        \n    return df    \n\n\ndef aggregate_exprs(df: pl.LazyFrame) -> List:\n    '''\n    Create aggregate expressions for tables with depth > 1. The\n    actual aggreation is done elsewhere following a groupby statement.\n\n    TODO: Currently only supports max aggregation. Add more expressions.\n    '''\n    \n    exprs = []\n    for column in df.columns:\n        if column[-1] in {'D', 'A', 'P'}:\n            # dates or numerical\n            exprs += [\n                pl.max(column).alias(f'max_{column}'),\n                pl.min(column).alias(f'min_{column}'),\n                pl.mean(column).alias(f'mean_{column}')\n            ]\n        # TODO: add more expressions\n    \n    return exprs\n\n\ndef read_file(path, depth:int=None) -> pl.LazyFrame:\n    '''\n    Read a parquet file and return a polars dataframe. If depth >= 1,\n    then the dataframe is grouped by case_id and aggregated.\n    '''\n    df = pl.scan_parquet(path).pipe(correct_dtypes)\n    \n    if depth in {1, 2}:\n        df = df.group_by('case_id').agg(aggregate_exprs(df))\n    \n    return df\n\ndef read_files(regex_path, depth:int=None) -> pl.LazyFrame:\n    '''\n    Read all parquet files matching the regex pattern and \n    concatenate them into a single dataframe. If depth >= 1,\n    then the dataframe is grouped by case_id and aggregated.\n    '''\n\n    chunks = []\n    for path in glob(str(regex_path)):\n        chunks.append(\n            pl.scan_parquet(path).pipe(correct_dtypes)\n        )\n\n    df = pl.concat(chunks, how='vertical_relaxed')\n    if depth in {1, 2}:\n        df = df.group_by('case_id').agg(aggregate_exprs(df))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:50:53.870860Z","iopub.execute_input":"2024-05-11T02:50:53.871243Z","iopub.status.idle":"2024-05-11T02:50:53.882148Z","shell.execute_reply.started":"2024-05-11T02:50:53.871216Z","shell.execute_reply":"2024-05-11T02:50:53.881275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Credit: https://www.kaggle.com/code/greysky/home-credit-baseline\ndata_stores = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"other_dfs\": [\n        # depth 0\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n        # depth 1\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n        # depth 2\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:50:55.385353Z","iopub.execute_input":"2024-05-11T02:50:55.385877Z","iopub.status.idle":"2024-05-11T02:50:55.600274Z","shell.execute_reply.started":"2024-05-11T02:50:55.385848Z","shell.execute_reply":"2024-05-11T02:50:55.599331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will now merge the dataframes to create one full training set. ","metadata":{}},{"cell_type":"code","source":"def datediff_col_exprs(date_columns):\n    exprs = []\n    for column in date_columns:\n        exprs.append(\n            (pl.col(column) - pl.col('date_decision')).dt.total_days().alias(f'days_from_decision_{column}')\n        )\n    \n    return exprs\n\ndef merge_and_basic_fe(df_base, other_dfs:List[pl.LazyFrame]):\n    '''\n    Combine the data and add some preliminary feature engineering\n    '''\n    \n    # add month and weekday columns for decision dates\n    df_base = (\n        df_base\n        # the MONTH seems to the same as date_decision without delimiters\n        .drop('MONTH') \n        .with_columns(\n            MONTH_NUM=pl.col('date_decision').dt.month(),\n            WEEKDAY_NUM=pl.col('date_decision').dt.weekday(),\n        )\n    )\n\n    # merge the other dataframes\n    for i, df in enumerate(other_dfs):\n        df_base = df_base.join(df, on='case_id', how='left', suffix=f'_{i}')\n    \n    # Create a new column for each date column that represents the number of \n    # days from the decision date. The orginal date column is then dropped.\n    schema = df_base.schema\n    date_columns = [column for column in schema if schema[column] == pl.Date]\n    df_base = (\n        df_base\n        .with_columns(datediff_col_exprs(date_columns))\n        .drop(date_columns)\n        .drop('date_decision')\n    )\n    \n    return df_base.with_columns(pl.col('opencred_647L').cast(pl.Int32))","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:50:57.284735Z","iopub.execute_input":"2024-05-11T02:50:57.285117Z","iopub.status.idle":"2024-05-11T02:50:57.293539Z","shell.execute_reply.started":"2024-05-11T02:50:57.285089Z","shell.execute_reply":"2024-05-11T02:50:57.292202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = merge_and_basic_fe(**data_stores)\nschema = df_train.schema\nprint(f'Number of columns........: {len(schema)}')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:50:59.472118Z","iopub.execute_input":"2024-05-11T02:50:59.472462Z","iopub.status.idle":"2024-05-11T02:50:59.486642Z","shell.execute_reply.started":"2024-05-11T02:50:59.472436Z","shell.execute_reply":"2024-05-11T02:50:59.485050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We haven't actually run anything until this step\n# We need to read in the data to calculate the number of observations\n# With Lazy API, polars will optimize the amount of data read into memory\nnum_obs = df_train.select(pl.len()).collect(streaming=True).item()\n\nprint(f'Number of observations...: {num_obs}')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:51:00.899315Z","iopub.execute_input":"2024-05-11T02:51:00.899657Z","iopub.status.idle":"2024-05-11T02:51:05.356009Z","shell.execute_reply.started":"2024-05-11T02:51:00.899632Z","shell.execute_reply":"2024-05-11T02:51:05.354903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will drop columns with missing values.","metadata":{}},{"cell_type":"code","source":"# 1. Compute the number of missing values in each column and compute the percentages\n# this op takes time to compute since we need to actually perform\n# operations on the entire training set\nmissing_perc = df_train.null_count().collect(streaming=True) / num_obs * 100\n\n# 2. Get a list of all columns with missing value percentages greater than 50%\ncolumns_to_drop = [col for col in missing_perc.columns if missing_perc[col][0] > 50]\nprint(f'Number of colums with > 50% missing values: {len(columns_to_drop)}')\n\n# 3. Drop these columns from the training set\ndf_train = df_train.drop(columns_to_drop)\nschema = df_train.schema\nprint(f'Number of columns after dropping: {len(schema)}')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:51:07.705652Z","iopub.execute_input":"2024-05-11T02:51:07.705980Z","iopub.status.idle":"2024-05-11T02:51:58.668940Z","shell.execute_reply.started":"2024-05-11T02:51:07.705955Z","shell.execute_reply":"2024-05-11T02:51:58.668109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are some columns which are categorical. We will drop columns that have only 1 unique value or have way too many unique values","metadata":{}},{"cell_type":"code","source":"cat_columns_to_drop = []\nfor column in schema:\n    if schema[column] == pl.String:\n        # get frequency \n        freq = df_train.select(column).unique().select(pl.len()).collect(streaming=True).item()\n        \n        if freq == 1 or freq > 20:\n            cat_columns_to_drop.append(column)\n\n# drop these columns\ndf_train = df_train.drop(cat_columns_to_drop)\n\nschema = df_train.schema\nprint(f'Number of columns after dropping: {len(schema)}')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:51:58.670389Z","iopub.execute_input":"2024-05-11T02:51:58.670896Z","iopub.status.idle":"2024-05-11T02:53:17.865332Z","shell.execute_reply.started":"2024-05-11T02:51:58.670862Z","shell.execute_reply":"2024-05-11T02:53:17.864489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA\n\nThe goal of the classification task is to predict the probability of a client defaulting on a loan. We will start by looking at the distribution of the target variable. There is a severe class imbalance in the target variable, with the majority class being the non-default class with about a 97% share of the data. ","metadata":{}},{"cell_type":"code","source":"target_dist = (\n    df_train\n    .group_by('target')\n    .agg(pl.count('target').alias('count'))\n    .collect(streaming=True)\n    .with_columns(\n        percentage = pl.col('count') / pl.col('count').sum() * 100\n    )\n    .to_pandas()\n    .set_index('target')\n)\ntarget_dist","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:54:23.185809Z","iopub.execute_input":"2024-05-11T02:54:23.186210Z","iopub.status.idle":"2024-05-11T02:54:27.215592Z","shell.execute_reply.started":"2024-05-11T02:54:23.186180Z","shell.execute_reply":"2024-05-11T02:54:27.214226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"By week number, we can see that the overall default rate in the training data fluctuates around 3% with some weeks having a default rate of 5% or more. There is no clear trend in the default rate over time. ","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(5, 3))\n(\n    df_train\n    .group_by('WEEK_NUM')\n    .agg(pl.mean('target') * 100)\n    .collect()\n    .sort(by='WEEK_NUM')\n    .to_pandas()\n    .plot(kind='line', x='WEEK_NUM', y='target', ax=ax, legend=False)\n)\n\n_ = ax.axhline(y=target_dist.loc[1, 'percentage'], color='k', linestyle='--', zorder=0)\n_ = ax.set_ylabel('Default percentage')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:55:03.741326Z","iopub.execute_input":"2024-05-11T02:55:03.741713Z","iopub.status.idle":"2024-05-11T02:55:05.888260Z","shell.execute_reply.started":"2024-05-11T02:55:03.741681Z","shell.execute_reply":"2024-05-11T02:55:05.887340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The number of queries to the credit bureau seems to have a positive relationship with the default rate. The default rate increases as the number of queries increases.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(6, 4))\n(\n    df_train\n    .group_by('numberofqueries_373L')\n    .agg(default_perc = pl.mean('target') * 100, n_obs = pl.len())\n    .collect(streaming=True)\n    .filter(pl.col('n_obs') > 250)\n    .sort(by='numberofqueries_373L')\n    .to_pandas()\n    .plot(kind='line', x='numberofqueries_373L', y='default_perc', ax=ax, legend=False)\n)\n\n_ = ax.axhline(y=target_dist.loc[1, 'percentage'], color='k', linestyle='--', zorder=0)\n_ = ax.set_ylabel('Default percentage')\n_ = ax.set_xlabel('Number of queries to credit bureau')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:55:24.812387Z","iopub.execute_input":"2024-05-11T02:55:24.812761Z","iopub.status.idle":"2024-05-11T02:55:29.118965Z","shell.execute_reply.started":"2024-05-11T02:55:24.812733Z","shell.execute_reply":"2024-05-11T02:55:29.117936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that in the plot above, we filtered out the number of queries that have less than 250 observations. All the filtered out cases have the number of queries > 37 (which is a lot).  Including these observations made the plot unreadable as there was too much noise - in some cases there were only 1 or 2 observations and these are likely to be outliers. To handle this, we will clip the number of queries at 38.","metadata":{}},{"cell_type":"code","source":"df_train = df_train.with_columns(\n    numberofqueries_373L = (\n        pl.when(pl.col('numberofqueries_373L') > 38).then(38)\n        .otherwise(pl.col('numberofqueries_373L'))\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:55:49.918213Z","iopub.execute_input":"2024-05-11T02:55:49.918606Z","iopub.status.idle":"2024-05-11T02:55:49.926008Z","shell.execute_reply.started":"2024-05-11T02:55:49.918575Z","shell.execute_reply":"2024-05-11T02:55:49.924914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert to pandas\n\nWe will convert the polars LazyFrame to a pandas DataFrame. As far as I know, lightgbm cannot directly process polars dataframes as of yet. Since some of the columns are categorical, we might need additional type processing for these columns","metadata":{}},{"cell_type":"code","source":"categorical_cols = [column for column in schema if schema[column] == pl.String]\n# expensive step: since \ndf_train = df_train.collect().to_pandas()\ndf_train[categorical_cols] = df_train[categorical_cols].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:55:59.626684Z","iopub.execute_input":"2024-05-11T02:55:59.627464Z","iopub.status.idle":"2024-05-11T02:56:14.090340Z","shell.execute_reply.started":"2024-05-11T02:55:59.627407Z","shell.execute_reply":"2024-05-11T02:56:14.088502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# garbage collection\ndel data_stores\n\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:56:14.094288Z","iopub.execute_input":"2024-05-11T02:56:14.094594Z","iopub.status.idle":"2024-05-11T02:56:14.214102Z","shell.execute_reply.started":"2024-05-11T02:56:14.094569Z","shell.execute_reply":"2024-05-11T02:56:14.213178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inspect pandas dataframe\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-11T02:56:14.215608Z","iopub.execute_input":"2024-05-11T02:56:14.216180Z","iopub.status.idle":"2024-05-11T02:56:14.243401Z","shell.execute_reply.started":"2024-05-11T02:56:14.216149Z","shell.execute_reply":"2024-05-11T02:56:14.242000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lightgbm Model\n\nCredit: https://www.kaggle.com/code/greysky/home-credit-baseline","metadata":{}},{"cell_type":"code","source":"X = df_train.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])\ny = df_train[\"target\"]\nweeks = df_train[\"WEEK_NUM\"]\n\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"device\": \"gpu\",\n}\n\nfitted_models = []\n\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(50)]\n    )\n\n    fitted_models.append(model)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save fitted models\nimport joblib\njoblib.dump(fitted_models, 'lgbm_fitted.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Credit: https://www.kaggle.com/code/greysky/home-credit-baseline\ndata_stores = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"other_dfs\": [\n        # depth 0\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n        # depth 1\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n        # depth 2\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n    ]\n}\n\n\n# process test set in appropriate formate\ndf_test = (\n    merge_and_basic_fe(**data_stores)\n    .drop(columns_to_drop) # drop columns with too many missing values\n    .drop(cat_columns_to_drop) # drop columns with too many categorical values\n    .with_columns(\n        numberofqueries_373L = (\n            pl.when(pl.col('numberofqueries_373L') > 38).then(38)\n            .otherwise(pl.col('numberofqueries_373L'))\n        )\n    )\n    .collect().to_pandas()\n    .set_index('case_id')\n    .drop(columns=['WEEK_NUM'])\n)\n\ndf_test[categorical_cols] = df_test[categorical_cols].astype(\"category\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate predictions (avgd probabilities)\nprob_preds = np.column_stack([\n    model.predict_proba(df_test)[:, 1] for model in fitted_models\n]).mean(axis=1)\n\nsubmission = pd.DataFrame({\n    'case_id': df_test.index.tolist(),\n    'score': prob_preds\n})\n\nsubmission.to_csv('submission.csv', index=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}