{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport regex as re\n\nfrom typing import Tuple, Set\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \nfrom sklearn.model_selection import GridSearchCV, KFold\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:39.865990Z","iopub.execute_input":"2024-04-03T16:09:39.866420Z","iopub.status.idle":"2024-04-03T16:09:44.209210Z","shell.execute_reply.started":"2024-04-03T16:09:39.866388Z","shell.execute_reply":"2024-04-03T16:09:44.207451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path_train = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/\"\ndata_path_test = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/\"","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:44.213057Z","iopub.execute_input":"2024-04-03T16:09:44.213733Z","iopub.status.idle":"2024-04-03T16:09:44.221122Z","shell.execute_reply.started":"2024-04-03T16:09:44.213660Z","shell.execute_reply":"2024-04-03T16:09:44.219433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category_pattern = r\"(?i)(isbid|iscre|status|state|cancel|education|code|profess|reject|body|sex|reason|district|empl_industry|role|relationshipto|name|language|gender|formdt|cat|lastst|twobod|description|maritalst|riskassesment|empl_employedtotal)\"\ndate_pattern = r\"(?i)(date|from|dtlas|firstcl|birth)\"\n  ","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:44.223413Z","iopub.execute_input":"2024-04-03T16:09:44.224238Z","iopub.status.idle":"2024-04-03T16:09:44.247476Z","shell.execute_reply.started":"2024-04-03T16:09:44.224181Z","shell.execute_reply":"2024-04-03T16:09:44.245580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"\n    Set data types for columns in a Polars DataFrame based on the last letter of their names.\n\n    Parameters:\n    - df (pl.DataFrame): The input DataFrame.\n\n    Returns:\n    - pl.DataFrame: The DataFrame with updated data types for specific columns.\n\n    Example:\n    ```\n    df = pl.DataFrame({\n        'columnP': [1, 2, 3],\n        'columnA': [4.5, 5.5, 6.5],\n        'columnB': ['a', 'b', 'c']\n    })\n    df = set_table_dtypes(df)\n    print(df)\n    ```\n    \"\"\"\n    for col in df.columns:\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n    return df\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Convert string/object columns in a Pandas DataFrame to categorical, adding \"Unknown\" to categories.\n\n    Parameters:\n    - df (pd.DataFrame): The input DataFrame.\n\n    Returns:\n    - pd.DataFrame: The DataFrame with string/object columns converted to categorical.\n\n    Example:\n    ```\n    df = pd.DataFrame({\n        'name': ['Alice', 'Bob', 'Charlie'],\n        'city': ['New York', 'Los Angeles', 'Chicago']\n    })\n    df = convert_strings(df)\n    print(df.dtypes)\n    ```\n    \"\"\"\n    for col in df.columns:  \n        if df[col].dtype.name in ['object', 'string']:\n            df[col] = df[col].astype(\"string\").astype('category')\n            current_categories = df[col].cat.categories\n            new_categories = current_categories.to_list() + [\"Unknown\"]\n            new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n            df[col] = df[col].astype(new_dtype)\n    return df\n\n\ndef transform_columns(df: pl.DataFrame, category_pattern: str, date_pattern: str) -> pl.DataFrame:\n    \"\"\"\n    Convert matching columns to either categorical or datetime format based on specific keywords in their names. \n    Columns with keywords indicating categorical data are converted to the categorical data type, with '_cat' \n    appended to their new names. Columns with keywords indicating dates are filled with a default date if null and \n    converted to datetime format, with '_datetime' appended to their new names. Original columns are dropped after conversion.\n\n    Parameters:\n    - df (pl.DataFrame): The input DataFrame.\n    - category_pattern (str): Regex pattern to match categorical columns.\n    - date_pattern (str): Regex pattern to match date columns.\n\n    Returns:\n    - pl.DataFrame: The DataFrame with specified columns converted to either categorical type or datetime format, \n                    excluding the original columns that were converted.\n    \"\"\"\n\n    # Pre-compile regex patterns for efficiency\n    category_regex = re.compile(category_pattern)\n    date_regex = re.compile(date_pattern)\n\n    transform_cols = []\n    keep_cols = []\n    \n    for col_name in df.columns:\n        if col_name == \"case_id\":\n            keep_cols.append(df[col_name])\n            continue\n\n        if col_name.endswith(\"A\"):\n            col = df[col_name].cast(pl.Utf8).cast(pl.Float64).alias(col_name)\n            transform_cols.append(col)\n            continue\n\n        # Handling categorical columns\n        if category_regex.search(col_name):\n            try:\n                if df[col_name].dtype in [pl.Int32, pl.Int64, pl.Float32, pl.Float64]:\n                    col = df[col_name].cast(pl.Utf8).cast(pl.Categorical).alias(f\"{col_name}\")\n                else:                \n                    col = df[col_name].cast(pl.Categorical).alias(f\"{col_name}\")\n                    column_categories = col.cat.get_categories()\n                    col = col.cat.to_local().to_physical().cast(pl.Int32).alias(f\"{col_name}\")\n                transform_cols.append(col)\n            except Exception as e:\n                print(f\"Error transforming categorical column {col_name}: {e}\")\n                keep_cols.append(df[col_name])\n\n        # Handling date columns\n        elif date_regex.search(col_name):\n            try:\n                col = pl.col(col_name).fill_null(\"2023-01-01\").str.strptime(pl.Date, \"%Y-%m-%d\", strict=False).alias(f\"{col_name}_datetime\")\n                transform_cols.append(col)\n            except Exception as e:\n                print(f\"Error transforming date column {col_name}: {e}\")\n                keep_cols.append(df[col_name])\n\n        # Keep columns as is if they don't match any pattern\n        else:\n            keep_cols.append(df[col_name])\n\n    # Combine transformed columns and columns to keep\n    df = df.select(keep_cols + transform_cols)\n    return df\n\ndef drop_date_columns(df: pl.DataFrame, date_pattern: str) -> pl.DataFrame:\n    '''\n    Drop columns from a Polars DataFrame that match a specific date pattern.\n\n    Parameters:\n    - df (pl.DataFrame): The input DataFrame.\n    - date_pattern (str): Regex pattern to match date columns.\n\n    Returns:\n    - pl.DataFrame: The DataFrame with date columns dropped.\n\n    '''\n    # Assuming date_regex is defined somewhere to match date column names\n    date_regex = re.compile(date_pattern)\n    \n    # Initialize lists to keep track of columns\n    keep_cols = []\n    drop_cols = []\n    \n    # Iterate through each column in the DataFrame\n    for col_name in df.columns:\n        # Check if the column name matches the date pattern\n        if date_regex.search(col_name):\n            # If it's a date column, prepare to drop it\n            drop_cols.append(col_name)\n        else:\n            # If it's not a date column, keep it\n            keep_cols.append(col_name)\n    \n    # Drop the date columns from the DataFrame\n    df = df.drop(drop_cols)\n    \n    return df\n\ndef compare_datasets(df1: pl.DataFrame, df2: pl.DataFrame) -> Tuple[Set[Tuple[str, str]], Set[Tuple[str, str]], Set[Tuple[str, str]]]:\n    \"\"\"\n    Compare two Polars DataFrames for their column names and data types, identifying differences and commonalities.\n\n    Parameters:\n    - df1 (pl.DataFrame): The first DataFrame for comparison.\n    - df2 (pl.DataFrame): The second DataFrame for comparison.\n\n    Returns:\n    - Tuple containing three sets for columns present only in df1, only in df2, and common columns, respectively, with their data types.\n\n    Example:\n    ```\n    df1 = pl.DataFrame({'A': [1, 2], 'B': ['x', 'y']})\n    df2 = pl.DataFrame({'B': ['x', 'y', 'z'], 'C': [True, False, True]})\n    diff1, diff2, common = compare_datasets(df1, df2)\n    print(\"Only in df1:\", diff1)\n    print(\"Only in df2:\", diff2)\n    print(\"Common columns:\", common)\n    ```\n    \"\"\"\n    df1_columns = [(name, str(dtype)) for name, dtype in zip(df1.columns, df1.dtypes)]\n    df2_columns = [(name, str(dtype)) for name, dtype in zip(df2.columns, df2.dtypes)]\n\n    df1_columns_set = set(df1_columns)\n    df2_columns_set = set(df2_columns)\n\n    columns_in_df1_not_in_df2 = df1_columns_set - df2_columns_set\n    columns_in_df2_not_in_df1 = df2_columns_set - df1_columns_set\n    common_columns = df1_columns_set & df2_columns_set\n\n    return columns_in_df1_not_in_df2, columns_in_df2_not_in_df1, common_columns\n\n\ndef remove_bool_null_string_dates(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype in [pl.Boolean, pl.Null, pl.String, pl.Date]:\n            df = df.drop(col)\n    return df\n\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    if not isinstance(base, pd.DataFrame):\n        raise ValueError(\"base must be a pandas DataFrame\")\n    \n    # Calculate the Gini coefficient over time\n    gini_in_time = base.sort_values(\"WEEK_NUM\").groupby(\"WEEK_NUM\")[[\"target\", \"score\"]].apply(lambda x: 2 * roc_auc_score(x[\"target\"], x[\"score\"]) - 1)\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time.values\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a * x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(y)\n    \n    plt.figure(figsize=(10, 6))\n    plt.plot(x, y, 'o', label='Gini over time')\n    plt.plot(x, y_hat, label='Fitted line', linestyle='--')\n    plt.xlabel('Week Number')\n    plt.ylabel('Gini Coefficient')\n    plt.title('Gini Coefficient over Time for Stability Score')\n    plt.legend()\n    plt.show()\n\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:44.251651Z","iopub.execute_input":"2024-04-03T16:09:44.252188Z","iopub.status.idle":"2024-04-03T16:09:44.310275Z","shell.execute_reply.started":"2024-04-03T16:09:44.252143Z","shell.execute_reply":"2024-04-03T16:09:44.308899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pl.read_parquet(data_path_train + \"train_base.parquet\")\ntrain_static = pl.concat(\n    [\n        pl.read_parquet(data_path_train + \"train_static_0_0.parquet\"),\n        pl.read_parquet(data_path_train + \"train_static_0_1.parquet\")\n    ],\n    how=\"vertical_relaxed\",\n)\ntrain_static_cb = pl.read_parquet(data_path_train + \"train_static_cb_0.parquet\")\ntrain_person_1 = pl.read_parquet(data_path_train + \"train_person_1.parquet\")\ntrain_applprev_1 = pl.read_parquet(data_path_train + \"train_applprev_1_0.parquet\")\ntrain_credit_bureau_b_2 = pl.read_parquet(data_path_train + \"train_credit_bureau_b_2.parquet\")\ntrain_debitcard_1 = pl.read_parquet(data_path_train + \"train_debitcard_1.parquet\")\ntrain_deposit_1 = pl.read_parquet(data_path_train+ \"train_deposit_1.parquet\")\ntrain_tax_registry = pl.read_parquet(data_path_train + \"train_tax_registry_b_1.parquet\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:44.312825Z","iopub.execute_input":"2024-04-03T16:09:44.313703Z","iopub.status.idle":"2024-04-03T16:09:56.642147Z","shell.execute_reply.started":"2024-04-03T16:09:44.313657Z","shell.execute_reply":"2024-04-03T16:09:56.640901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_parquet(data_path_test + \"test_base.parquet\")\ntest_static = pl.concat(\n    [\n        pl.read_parquet(data_path_test + \"test_static_0_0.parquet\"),\n        pl.read_parquet(data_path_test +  \"test_static_0_1.parquet\"),\n        pl.read_parquet(data_path_test + \"test_static_0_2.parquet\")\n    ],\n    how=\"vertical_relaxed\",\n)\ntest_static_cb = pl.read_parquet(data_path_test + \"test_static_cb_0.parquet\")\ntest_person_1 = pl.read_parquet(data_path_test + \"test_person_1.parquet\")\ntest_applprev_1 = pl.read_parquet(data_path_train + \"train_applprev_1_0.parquet\")\ntest_credit_bureau_b_2 = pl.read_parquet(data_path_test + \"test_credit_bureau_b_2.parquet\")\ntest_debitcard_1 = pl.read_parquet(data_path_test + \"test_debitcard_1.parquet\")\ntest_deposit_1 = pl.read_parquet(data_path_test + \"test_deposit_1.parquet\")\ntest_tax_registry = pl.read_parquet(data_path_test + \"test_tax_registry_b_1.parquet\")","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:56.645181Z","iopub.execute_input":"2024-04-03T16:09:56.645628Z","iopub.status.idle":"2024-04-03T16:09:59.900293Z","shell.execute_reply.started":"2024-04-03T16:09:56.645597Z","shell.execute_reply":"2024-04-03T16:09:59.898740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\ntrain_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"num_group_0_person_housetype\"})\n\ntrain_person_1_feats_3 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 1\n).drop(\"num_group1\").rename({\"housetype_905L\": \"num_group_1_person_housetype\"})\n\ntrain_person_1_feats_4 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 2\n).drop(\"num_group1\").rename({\"housetype_905L\": \"num_group_2_person_housetype\"})\n\ntrain_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\ntrain_static_annuity = train_static.group_by([\"case_id\"]).agg(['annuity_780A']).sum()\n\n# We will process in this examples only A-type and M-type columns, so we need to select them.\nselected_static_cols = []\nfor col in train_static.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cols.append(col)\nprint(selected_static_cols)\n\nselected_static_cb_cols = []\nfor col in train_static_cb.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cb_cols.append(col)\nprint(selected_static_cb_cols)\n\nselected_applprev_1_cols = []\nfor col in train_applprev_1.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_applprev_1_cols.append(col)\nprint(selected_applprev_1_cols)\n\nselected_credit_bureau_b_2_cols = []\nfor col in train_credit_bureau_b_2.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_credit_bureau_b_2_cols.append(col)\nprint(selected_credit_bureau_b_2_cols)\n\nselected_debitcard_1_cols = []\nfor col in train_debitcard_1.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_debitcard_1_cols.append(col)\nprint(selected_debitcard_1_cols)\n\nselected_deposit_1_cols = []\nfor col in train_deposit_1.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_deposit_1_cols.append(col)\nprint(selected_deposit_1_cols)\n\ndata = train_basetable.join(\n    train_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_3, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_4, how=\"left\", on=\"case_id\"\n).join(\n    train_applprev_1.select([\"case_id\"] + selected_applprev_1_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    train_debitcard_1.select([\"case_id\"] + selected_debitcard_1_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_deposit_1.select([\"case_id\"] + selected_deposit_1_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_tax_registry.select([\"case_id\", \"num_group1\"]), how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:09:59.902393Z","iopub.execute_input":"2024-04-03T16:09:59.902998Z","iopub.status.idle":"2024-04-03T16:10:38.890219Z","shell.execute_reply.started":"2024-04-03T16:09:59.902924Z","shell.execute_reply":"2024-04-03T16:10:38.887906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.group_by(\"case_id\").agg([pl.col(pl.Float64).mean(), pl.col(pl.Float32).mean(), pl.col(pl.Int64).mean(), pl.col(pl.Int32).mean()])","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:10:38.894175Z","iopub.execute_input":"2024-04-03T16:10:38.896586Z","iopub.status.idle":"2024-04-03T16:10:50.431097Z","shell.execute_reply.started":"2024-04-03T16:10:38.896497Z","shell.execute_reply":"2024-04-03T16:10:50.428513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = transform_columns(data, category_pattern, date_pattern)\ndata = remove_bool_null_string_dates(data).fill_nan(0).fill_null(0)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:10:50.433354Z","iopub.execute_input":"2024-04-03T16:10:50.433799Z","iopub.status.idle":"2024-04-03T16:11:00.685414Z","shell.execute_reply.started":"2024-04-03T16:10:50.433764Z","shell.execute_reply":"2024-04-03T16:11:00.684014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata.shape\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:00.689618Z","iopub.execute_input":"2024-04-03T16:11:00.690454Z","iopub.status.idle":"2024-04-03T16:11:00.698687Z","shell.execute_reply.started":"2024-04-03T16:11:00.690410Z","shell.execute_reply":"2024-04-03T16:11:00.697488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"num_group_0_person_housetype\"})\n\ntest_person_1_feats_3 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 1\n).drop(\"num_group1\").rename({\"housetype_905L\": \"num_group_1_person_housetype\"})\n\ntest_person_1_feats_4 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 2\n).drop(\"num_group1\").rename({\"housetype_905L\": \"num_group_2_person_housetype\"})\n\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n\n# We will process in this examples only A-type and M-type columns, so we need to select them.\nselected_static_cols = []\nfor col in test_static.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cols.append(col)\nprint(selected_static_cols)\n\nselected_static_cb_cols = []\nfor col in test_static_cb.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_static_cb_cols.append(col)\nprint(selected_static_cb_cols)\n\nselected_applprev_1_cols = []\nfor col in test_applprev_1.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_applprev_1_cols.append(col)\nprint(selected_applprev_1_cols)\n\nselected_credit_bureau_b_2_cols = []\nfor col in test_credit_bureau_b_2.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_credit_bureau_b_2_cols.append(col)\nprint(selected_credit_bureau_b_2_cols)\n\nselected_debitcard_1_cols = []\nfor col in test_debitcard_1.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_debitcard_1_cols.append(col)\nprint(selected_debitcard_1_cols)\n\nselected_deposit_1_cols = []\nfor col in test_deposit_1.columns:\n    if col[-1] in (\"A\", \"M\"):\n        selected_deposit_1_cols.append(col)\nprint(selected_deposit_1_cols)\n\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_3, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_4, how=\"left\", on=\"case_id\"\n).join(\n    test_applprev_1.select([\"case_id\"] + selected_applprev_1_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    test_debitcard_1.select([\"case_id\"] + selected_debitcard_1_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_deposit_1.select([\"case_id\"] + selected_deposit_1_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_tax_registry.select([\"case_id\", \"num_group1\"]), how=\"left\", on=\"case_id\"\n)\n ","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:00.700553Z","iopub.execute_input":"2024-04-03T16:11:00.701279Z","iopub.status.idle":"2024-04-03T16:11:01.188968Z","shell.execute_reply.started":"2024-04-03T16:11:00.701236Z","shell.execute_reply":"2024-04-03T16:11:01.187969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submission = data_submission.group_by(\"case_id\").agg([pl.col(pl.Float64).mean(), pl.col(pl.Float32).mean(), pl.col(pl.Int64).mean(), pl.col(pl.Int32).mean()])","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:01.190474Z","iopub.execute_input":"2024-04-03T16:11:01.191043Z","iopub.status.idle":"2024-04-03T16:11:01.197825Z","shell.execute_reply.started":"2024-04-03T16:11:01.191009Z","shell.execute_reply":"2024-04-03T16:11:01.196961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = data.to_pandas()\ndata_submission = data_submission.to_pandas()\n\n# Split data into features (X) and target (y), excluding 'case_id' to prevent data leakage\nX = data.drop('target', axis=1)\ny = data['target']\n\n# Identify categorical features\ncategorical_features = [col for col in X.columns if col.endswith('_cat')]\n\n# Adjusted split to include a validation set\nX_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.3, random_state=42)  # Increase test_size to carve out validation set\nX_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42)  # Split the remaining data equally into validation and test sets\n\n# Prepare LightGBM Datasets\ntrain_data = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_features, free_raw_data=True)\ntrain_data.save_binary('train_data.bin')\nval_data = lgb.Dataset(X_val, label=y_val, reference=train_data, categorical_feature=categorical_features, free_raw_data=True)\nval_data.save_binary('val_data.bin')\ntest_data = lgb.Dataset(X_test, label=y_test, reference=train_data, categorical_feature=categorical_features, free_raw_data=True)\ntest_data.save_binary('test_data.bin')\n\ntrain_data = lgb.Dataset('train_data.bin')\nval_data = lgb.Dataset('val_data.bin', reference=train_data)\ntest_data = lgb.Dataset('test_data.bin', reference=train_data)\n# Define parameters for the model\nparams = {\n    'objective': 'binary',                # Objective function\n    'metric': 'auc',                      # Evaluation metric\n    'boosting_type': 'gbdt',              # Gradient Boosting Decision Tree\n    'learning_rate': 0.08,                 # Learning rate, controls the step size in the parameter space search\n    'num_leaves': 48,                     # Maximum tree leaves for base learners\n    'max_depth': -1,                      # Maximum tree depth for base learners, -1 means no limit\n    'min_data_in_leaf': 20,               # Minimum number of data points in one leaf\n    'feature_fraction': 0.8,              # Randomly select a fraction of features on each iteration (tree)\n    'subsample': 0.8,                     # Subsample ratio of the training instance\n    'subsample_freq': 1,                  # Frequence of subsample, <=0 means no enable\n    'colsample_bytree': 0.8,              # Subsample ratio of columns when constructing each tree\n    'min_child_weight': 0.001,            # Minimum sum of instance weight (hessian) needed in a child (leaf)\n    'min_split_gain': 0.0,                # Minimum loss reduction required to make a further partition on a leaf node of the tree\n    'reg_alpha': 0.0,                     # L1 regularization term on weights\n    'reg_lambda': 1.0,                    # L2 regularization term on weights\n    'seed': 42,                           # Random seed for reproducibility\n    'bagging_freq': 5                     # Random seed for bagging\n}\n\n# Train the model with validation set\nbase_valid = [val_data, test_data]  # Including validation data in the training process\nbst = lgb.train(params, train_data, valid_sets=base_valid, early_stopping_rounds=10, verbose_eval=10)\n\n# Predict and evaluate for the test set\ntest_predictions = bst.predict(X_test, predict_disable_shape_check=True)\ntest_auc_score = roc_auc_score(y_test, test_predictions)\nprint(f\"AUC Score (Test): {test_auc_score}\")\n\n\n# Assuming 'data_submission' is to be used as an additional evaluation step\ndata_submission_features = data_submission\ndata_submission['score'] = bst.predict(data_submission_features, predict_disable_shape_check=True)\nif 'target' in data_submission.columns:\n    data_submission_auc_score = roc_auc_score(data_submission['target'], data_submission['score'])\n    print(f\"AUC Score (Validation): {data_submission_auc_score}\")\n\n# Save the model\nbst.save_model('lightgbm_model.txt')","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:01.199917Z","iopub.execute_input":"2024-04-03T16:11:01.200337Z","iopub.status.idle":"2024-04-03T16:11:33.345608Z","shell.execute_reply.started":"2024-04-03T16:11:01.200305Z","shell.execute_reply":"2024-04-03T16:11:33.344325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot feature importance\nplt.figure(figsize=(30, 10))\nlgb.plot_importance(bst, importance_type='split', max_num_features=40, title=\"Feature Importance (Split)\")\nplt.show()\n\nplt.figure(figsize=(30, 10))\nlgb.plot_importance(bst, importance_type='gain', max_num_features=40, title=\"Feature Importance (Gain)\")\nplt.show()\n\n# Save the model\nbst.save_model('lightgbm_model.txt')","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:33.347674Z","iopub.execute_input":"2024-04-03T16:11:33.348498Z","iopub.status.idle":"2024-04-03T16:11:34.500306Z","shell.execute_reply.started":"2024-04-03T16:11:33.348447Z","shell.execute_reply":"2024-04-03T16:11:34.499383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate predictions (scores) for each set\n# Note: We'll directly generate series of scores without altering the original X DataFrames\nscores_train = bst.predict(X_train, predict_disable_shape_check=True)\nscores_test = bst.predict(X_test, predict_disable_shape_check=True)\n# If you have a validation set, generate scores for it as well\nscores_val = bst.predict(X_val, predict_disable_shape_check=True)\n\n# Combine predictions with the true target values and WEEK_NUM for training set\nbase_train = X_train.copy()\nbase_train['score'] = scores_train\nbase_train['target'] = y_train\n# Make sure 'WEEK_NUM' is included if not already in X_train\n\n# Similar operation for X_test and y_test to create base_test\nbase_test = X_test.copy()\nbase_test['score'] = scores_test\nbase_test['target'] = y_test\n# Ensure 'WEEK_NUM' is included if not already in X_test\n\n# If you have a validation set, prepare it similarly\nbase_val = X_val.copy()\nbase_val['score'] = scores_val\nbase_val['target'] = y_val\n\n# Make sure 'WEEK_NUM' is included if not already in X_val\n\n# Calculate Gini stability scores\n# Ensure datasets are correctly prepared with 'WEEK_NUM', 'target', and 'score'\nstability_score_train = gini_stability(base_train)  # Assuming 'WEEK_NUM' is properly set\nstability_score_validation = gini_stability(base_val)  # Uncomment if you have a validation set\nstability_score_test = gini_stability(base_test)\n\nprint(f'Stability Score (Train): {stability_score_train}')\nprint(f'Stability Score (Validation): {stability_score_validation}')  # Uncomment if you have a validation set\nprint(f'Stability Score (Test): {stability_score_test}')","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:34.501713Z","iopub.execute_input":"2024-04-03T16:11:34.502295Z","iopub.status.idle":"2024-04-03T16:11:44.077366Z","shell.execute_reply.started":"2024-04-03T16:11:34.502262Z","shell.execute_reply":"2024-04-03T16:11:44.076483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_submission_path = \"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\"\nsave_file_path_dir = \"/kaggle/working/submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:11:44.078926Z","iopub.execute_input":"2024-04-03T16:11:44.079640Z","iopub.status.idle":"2024-04-03T16:11:44.084548Z","shell.execute_reply.started":"2024-04-03T16:11:44.079605Z","shell.execute_reply":"2024-04-03T16:11:44.083462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Step 1: Load the CSV file\nfile_path = file_submission_path # Adjust path if necessary\ndata_predictions = pd.read_csv(file_path)\n\n# Assuming 'data_submission' DataFrame contains 'case_id' and the necessary features for prediction\n# Merge or map 'case_id's from 'data_predictions' to 'data_submission' to get the corresponding features\nfeatures_df = pd.merge(data_predictions[['case_id']], data_submission, on='case_id', how='left')\n\n# Drop the 'case_id' column to prepare features for prediction\nfeatures = features_df.drop(columns=['case_id'], errors='ignore')\n\n# Generate predictions\npredictions = bst.predict(features, predict_disable_shape_check=True)\n\n# Create the submission DataFrame\nsubmission_df = pd.DataFrame({\n    'case_id': features_df['case_id'],  # Use the case_id from the merged DataFrame\n    'score': predictions\n})\n\n# Save the submission DataFrame to a new CSV file\nsave_file_path = save_file_path_dir\nsubmission_file_path = save_file_path_dir  # Adjust the path as necessary for your environment\nsubmission_df.to_csv(submission_file_path)\n\n# Output the path to the saved submission file\nsubmission_file_path","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:12:37.026150Z","iopub.execute_input":"2024-04-03T16:12:37.026680Z","iopub.status.idle":"2024-04-03T16:12:37.051661Z","shell.execute_reply.started":"2024-04-03T16:12:37.026639Z","shell.execute_reply":"2024-04-03T16:12:37.050783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-04-03T16:12:40.030333Z","iopub.execute_input":"2024-04-03T16:12:40.030997Z","iopub.status.idle":"2024-04-03T16:12:40.056731Z","shell.execute_reply.started":"2024-04-03T16:12:40.030912Z","shell.execute_reply":"2024-04-03T16:12:40.054467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}