{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30684,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Gradient Boosting Regressor Test\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\n#import lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nimport pyarrow as pa\nfrom sklearn.ensemble import GradientBoostingRegressor\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:43:05.061451Z","iopub.execute_input":"2024-05-12T21:43:05.061929Z","iopub.status.idle":"2024-05-12T21:43:05.068451Z","shell.execute_reply.started":"2024-05-12T21:43:05.061899Z","shell.execute_reply":"2024-05-12T21:43:05.067237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    # implement here all desired dtypes for tables\n    # the following is just an example\n    for col in df.columns:\n        # last letter of column name will help you determine the type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:43:07.609359Z","iopub.execute_input":"2024-05-12T21:43:07.609781Z","iopub.status.idle":"2024-05-12T21:43:07.619712Z","shell.execute_reply.started":"2024-05-12T21:43:07.609749Z","shell.execute_reply":"2024-05-12T21:43:07.618477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_csv(dataPath + \"csv_files/train/train_base.csv\")\ntrain_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_csv(dataPath + \"csv_files/train/train_static_cb_0.csv\").pipe(set_table_dtypes)\ntrain_person_1 = pl.read_csv(dataPath + \"csv_files/train/train_person_1.csv\").pipe(set_table_dtypes) \ntrain_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_b_2.csv\").pipe(set_table_dtypes)\ntrain_tax_registry_a_1 = pl.read_csv(dataPath + \"csv_files/train/train_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntrain_credit_bureau_a_1 = pl.read_csv(dataPath + \"csv_files/train/train_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes).select(\"case_id\",\"debtoutstand_525A\",\"debtoverdue_47A\",\"totaldebtoverduevalue_178A\")","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:43:10.68203Z","iopub.execute_input":"2024-05-12T21:43:10.684524Z","iopub.status.idle":"2024-05-12T21:43:39.259696Z","shell.execute_reply.started":"2024-05-12T21:43:10.684469Z","shell.execute_reply":"2024-05-12T21:43:39.258628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#müll analyse zelle\n#train_static.head()\n#train_person_1.head()\n#train_tax_registry_a_1.head()\n# train_credit_bureau_a_1 = train_credit_bureau_a_1.with_columns(\n#     pl.col(\"debtoutstand_525A\") + pl.col(\"debtoverdue_47A\") + pl.col(\"totaldebtoverduevalue_178A\").alias(\"totaldebt\")\n# )\n# train_credit_bureau_a_1.head()\n# train_credit_bureau_a_1_feats = train_credit_bureau_a_1.group_by(\"case_id\").agg(pl.col(\"totaldebt\").max().alias(\"maxtotaldebt\"))\n# train_credit_bureau_a_1_feats.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#müll analyse zelle\n# train_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n#     pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n#     (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n# )\n# train_person_1_feats_1.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_csv(dataPath + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n        pl.read_csv(dataPath + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes),\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static_cb = pl.read_csv(dataPath + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1 = pl.read_csv(dataPath + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes) \ntest_credit_bureau_b_2 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes) \ntest_tax_registry_a_1 = pl.read_csv(dataPath + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_a_1 = pl.read_csv(dataPath + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:43:44.179407Z","iopub.execute_input":"2024-05-12T21:43:44.179817Z","iopub.status.idle":"2024-05-12T21:43:44.273829Z","shell.execute_reply.started":"2024-05-12T21:43:44.179785Z","shell.execute_reply":"2024-05-12T21:43:44.272775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature engineering\n\nIn this part, we can see a simple example of joining tables via `case_id`. Here the loading and joining is done with polars library. Polars library is blazingly fast and has much smaller memory footprint than pandas. ","metadata":{}},{"cell_type":"code","source":"# We need to use aggregation functions in tables with depth > 1, so tables that contain num_group1 column or \n# also num_group2 column.\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# Here num_group1=0 has special meaning, it is the person who applied for the loan.\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Here we have num_goup1 and num_group2, so we need to aggregate again.\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n# #Train Credit Bureau A 1 Features:\n# train_credit_bureau_a_1 = train_credit_bureau_a_1.with_columns(\n#     (pl.col(\"debtoutstand_525A\") + pl.col(\"debtoverdue_47A\") + pl.col(\"totaldebtoverduevalue_178A\")).alias(\"totaldebt\")\n# )\n\n# train_credit_bureau_a_1_feats = train_credit_bureau_a_1.group_by(\"case_id\").agg([\n#     pl.col(\"case_id\"),  # include case_id in the aggregation result\n#     pl.col(\"totaldebt\").max().alias(\"maxtotaldebt\")\n# ])\n# train_credit_bureau_a_1_feats = train_credit_bureau_a_1_feats.join(train_person_1_feats_1.select(\"mainoccupationinc_384A_max\"), how=\"left\", on=\"case_id\")\n\n\n# #Here we try to engineer ratios from different columns of the credit bureau data\n# train_credit_bureau_a_1_feats[\"debt_to_income_ratio\"] = train_credit_bureau_a_1_feats['maxtotaldebt'] / train_credit_bureau_a_1_feats[\"mainoccupationinc_384A_max\"]\n\n# We will process in this examples only A-type and M-type columns, so we need to select them.\nselected_static_cols = []\nfor col in train_static.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cols.append(col)\nprint(selected_static_cols)\n\nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cb_cols.append(col)\nprint(selected_static_cb_cols)\n\n# Join all tables together.\ndata = train_basetable.join(\n    train_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(train_tax_registry_a_1, how=\"left\", on=\"case_id\"\n).join(train_credit_bureau_a_1, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:43:47.186632Z","iopub.execute_input":"2024-05-12T21:43:47.187089Z","iopub.status.idle":"2024-05-12T21:43:54.383597Z","shell.execute_reply.started":"2024-05-12T21:43:47.187052Z","shell.execute_reply":"2024-05-12T21:43:54.382089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:44:01.086834Z","iopub.execute_input":"2024-05-12T21:44:01.087268Z","iopub.status.idle":"2024-05-12T21:44:01.114804Z","shell.execute_reply.started":"2024-05-12T21:44:01.087236Z","shell.execute_reply":"2024-05-12T21:44:01.113468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    test_tax_registry_a_1, how=\"left\", on=\"case_id\").join(test_credit_bureau_a_1, how=\"left\", on=\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:45:53.557928Z","iopub.execute_input":"2024-05-12T21:45:53.55899Z","iopub.status.idle":"2024-05-12T21:45:53.579825Z","shell.execute_reply.started":"2024-05-12T21:45:53.558944Z","shell.execute_reply":"2024-05-12T21:45:53.578851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_ids = data[\"case_id\"].unique().shuffle(seed=1)\ncase_ids_train, case_ids_test = train_test_split(case_ids.to_list(), train_size=0.9, random_state=1)\ncase_ids_valid, case_ids_test = train_test_split(case_ids_test, train_size=0.9, random_state=1)\n\ncols_pred = []\nfor col in data.columns:\n    if col[-1].isupper() and col[:-1].islower():\n        cols_pred.append(col)\n\nprint(cols_pred)\n\ndef from_polars_to_pandas(case_ids: pl.DataFrame) -> pl.DataFrame:\n    return (\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[cols_pred].to_pandas(),\n        data.filter(pl.col(\"case_id\").is_in(case_ids))[\"target\"].to_pandas()\n    )\n\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\nfor df in [X_train, X_valid, X_test]:\n    df = convert_strings(df)","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:45:58.100391Z","iopub.execute_input":"2024-05-12T21:45:58.100821Z","iopub.status.idle":"2024-05-12T21:46:48.994502Z","shell.execute_reply.started":"2024-05-12T21:45:58.100788Z","shell.execute_reply":"2024-05-12T21:46:48.993443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:47:56.832988Z","iopub.execute_input":"2024-05-12T21:47:56.833401Z","iopub.status.idle":"2024-05-12T21:47:56.840417Z","shell.execute_reply.started":"2024-05-12T21:47:56.83337Z","shell.execute_reply":"2024-05-12T21:47:56.839172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train: {X_train.shape}\")\nprint(f\"Valid: {X_valid.shape}\")\nprint(f\"Test: {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:47:59.193476Z","iopub.execute_input":"2024-05-12T21:47:59.193861Z","iopub.status.idle":"2024-05-12T21:47:59.200775Z","shell.execute_reply.started":"2024-05-12T21:47:59.193832Z","shell.execute_reply":"2024-05-12T21:47:59.19941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training LightGBM\n\nMinimal example of LightGBM training is shown below.","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\n\n# Define which columns need each type of preprocessing\nnumeric_features = X_train.select_dtypes(include=['int64', 'float64']).columns\ncategorical_features = X_train.select_dtypes(include=['object']).columns\n\n# Create transformers for each type of preprocessing\nnumeric_transformer = SimpleImputer(strategy='mean')\ncategorical_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='most_frequent')),\n    ('encoder', LabelEncoder())\n])\n\n# Combine transformers\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', numeric_transformer, numeric_features),\n        ('cat', categorical_transformer, categorical_features)\n    ])\n\n# Create the pipeline\npipeline = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('regressor', GradientBoostingRegressor(\n        max_depth=3,\n        n_estimators=375,\n        learning_rate=0.05,\n        min_samples_split=5,\n        verbose=1\n    ))\n])\n# Make a copy of y_train\ny_train_copy = y_train.copy()\n\n# Train the model with the transformed training data\npipeline.fit(X_train, y_train_copy)\n\n# Make predictions on the validation data\npredictions = pipeline.predict(X_valid)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-12T21:48:28.349395Z","iopub.execute_input":"2024-05-12T21:48:28.349791Z","iopub.status.idle":"2024-05-12T21:50:10.110675Z","shell.execute_reply.started":"2024-05-12T21:48:28.349761Z","shell.execute_reply":"2024-05-12T21:50:10.107219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!lscpu |grep 'Model name'\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Evaluation with AUC and then comparison with the stability metric is shown below.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\n# Predictions für Trainings-, Validierungs- und Testdaten machen\ny_train_pred = pipeline.predict(X_train)\ny_valid_pred = pipeline.predict(X_valid)\ny_test_pred = pipeline.predict(X_test)\n\n# AUC-Scores berechnen\nauc_train = roc_auc_score(y_train.copy(), y_train_pred)\nauc_valid = roc_auc_score(y_valid.copy(), y_valid_pred)\nauc_test = roc_auc_score(y_test.copy(), y_test_pred)\n\n\nprint(f'The AUC score on the train set is: {auc_train}') \nprint(f'The AUC score on the valid set is: {auc_valid}') \nprint(f'The AUC score on the test set is: {auc_test}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\n\nScoring the submission dataset is below, we need to take care of new categories. Then we save the score as a last step. ","metadata":{}},{"cell_type":"code","source":"X_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\ncategorical_cols = X_train.select_dtypes(include=['category']).columns\n\n\n# Loop über kategoriale Spalten\nfor col in categorical_cols:\n    train_categories = set(X_train[col].cat.categories)\n    submission_categories = set(X_submission[col].cat.categories)\n    \n    # Identifiziere neue Kategorien in den Submissions-Daten\n    new_categories = submission_categories - train_categories\n    \n    # Ersetze neue Kategorien durch \"Unknown\"\n    X_submission.loc[X_submission[col].isin(new_categories), col] = \"Unknown\"\n    \n    # Definiere den Datentyp der kategorialen Spalte neu, um sicherzustellen, dass beide Datensätze dieselben Kategorien haben\n    new_dtype = pd.CategoricalDtype(categories=train_categories, ordered=True)\n    X_submission[col] = X_submission[col].astype(new_dtype)\n\n# Mache Vorhersagen auf den Submissions-Daten\ny_submission_pred = pipeline.predict(X_submission)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": data_submission[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nsubmission.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Analysis**\n\nHere we perform a permutation importance to gain insights as to what features are the most important for the model.","metadata":{}},{"cell_type":"code","source":"from sklearn.inspection import permutation_importance\n\n#result = permutation_importance(GradientBoostingRegressor, X_test, y_test, n_repeats=10, random_state=42)\n#importances = result.importances_mean\n#std = result.importances_std\n\n#print(result)\n#print(importances)\n#print(std) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}