{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30715,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport lightgbm as lgb \nimport numpy as np  \nimport pandas as pd  \nimport polars as pl  \nimport warnings\n\nfrom catboost import CatBoostClassifier, Pool  \nfrom glob import glob\nfrom IPython.display import display  \nfrom pathlib import Path\nfrom sklearn.base import BaseEstimator, ClassifierMixin  \nfrom sklearn.metrics import roc_auc_score  \nfrom sklearn.model_selection import StratifiedGroupKFold  \nfrom typing import Any\n\n\n\nwarnings.filterwarnings(\"ignore\")\nROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\n\nTRAIN_DIR = ROOT/ \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-07T09:41:50.099217Z","iopub.execute_input":"2024-06-07T09:41:50.099880Z","iopub.status.idle":"2024-06-07T09:41:50.109106Z","shell.execute_reply.started":"2024-06-07T09:41:50.099846Z","shell.execute_reply":"2024-06-07T09:41:50.107807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SchemaGen:\n    \n    @staticmethod\n    def change_dtypes(df : pl.LazyFrame) -> pl.LazyFrame:\n        \n        for col in df.columns:\n            if col == \"case_id\":\n                df = df.with_columns(pl.col(col).cast(pl.UInt32).alias(col))\n            elif col in [\"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.UInt16).alias(col))\n            elif col == \"date_decision\" or col[-1] == \"D\":\n                df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n            elif col[-1] in [\"P\", \"A\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            elif col[-1] in [\"M\"]:\n                df = df.with_columns(pl.col(col).cast(pl.String).alias(col))\n        return df\n    \n    @staticmethod\n    def scan_files(global_path : str, depth : int = None) -> pl.LazyFrame:\n        \n        chunks : list[pl.LazyFrame] = []\n        \n        for path in glob(str(global_path)):\n            df:pl.LazyFrame = pl.scan_parquet(path, low_memory = True, rechunk = True).pipe(SchemaGen.change_dtypes)\n            print(f\"File {Path(path).stem} loaded into memory\")\n            \n            if depth in (1, 2):\n                exprs : list[pl.Series] = Aggregator.get_exprs(df)\n                df = df.group_by(\"case_id\").agg(exprs)\n                \n                del exprs\n                gc.collect()\n                \n            chunks.append(df)\n        \n        df = pl.concat(chunks , how = \"vertical_relaxed\")\n        del chunks\n        gc.collect()\n        \n        df = df.unique(subset = [\"case_id\"])\n        \n        return df\n    \n    def join_dataframes(\n    df_base : pl.LazyFrame,\n    depth_0 = list[pl.LazyFrame],\n    depth_1 = list[pl.LazyFrame],\n    depth_2 = list[pl.LazyFrame]\n    ) -> pl.DataFrame :\n        \n        for i, df in enumerate(depth_0 + depth_1 + depth_2):\n            df_base = df_base.join(df, how = \"left\", on = \"case_id\", suffix = f\"_{i}\")\n            \n        return df_base.collect().pipe(Utility.reduce_memory_usage, \"df_train\")\n    \n        ","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:09:25.554481Z","iopub.execute_input":"2024-06-07T08:09:25.555206Z","iopub.status.idle":"2024-06-07T08:09:25.573827Z","shell.execute_reply.started":"2024-06-07T08:09:25.555140Z","shell.execute_reply":"2024-06-07T08:09:25.572325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    \n    @staticmethod\n    def max_expr(df : pl.LazyFrame) -> list[pl.Series]:\n        cols : list[str] = [\n            col \n            for col in df.columns\n            if (col[-1] in (\"P\", \"M\", \"A\", \"D\", \"T\", \"L\")) or (\"num_group\" in col)\n        ]\n        expr_max : list[pl.Series] = [\n            pl.col(col).max().alias(f\"max_{col}\") for col in cols\n        ]\n        return expr_max\n    \n    @staticmethod\n    def min_expr(df : pl.LazyFrame) -> list[pl.Series]:\n        \n        cols : list[str] = [\n            col\n            for col in df.columns\n            if (col[-1] in (\"P\", \"M\", \"A\", \"D\", \"T\", \"L\")) or (\"num_group\" in col)\n        ]\n        \n        expr_min : list[pl.Series] = [\n            pl.col(col).min().alias(f\"min_{col}\") for col in cols\n        ]\n        return expr_min\n    \n    @staticmethod\n    def mean_expr(df : pl.LazyFrame) -> list[pl.Series]:\n        cols : list[str] = [\n            col\n            for col in df.columns\n            if col.endswith((\"P\", \"A\", \"D\"))\n        ]\n        \n        expr_mean : list[pl.Series] = [\n            pl.col(col).mean().alias(f\"mean_{col}\") for col in cols\n        ]\n            \n        return expr_mean\n            \n    @staticmethod\n    def var_expr(df : pl.LazyFrame) -> list[pl.Series]:\n        cols : list[str] = [col for col in df.columns if col.endswith((\"P\", \"A\", \"D\"))]\n            \n        expr_var : list[pl.Series] = [\n            pl.col(col).var().alias(f\"var_{col}\") for col in cols\n        ]\n        \n        return expr_var\n    \n    @staticmethod\n    def mode_expr(df : pl.LazyFrame) -> list[pl.Series]:\n        cols : list[str] = [col for col in df.columns if col.endswith(\"M\")]\n        \n        expr_mode : list[pl.Series] = [\n            pl.col(col).mode().alias(f\"mode_{col}\") for col in cols\n        ]\n    \n        return expr_mode\n    \n    @staticmethod\n    def get_exprs(df : pl.LazyFrame) -> list[pl.Series]:\n        exprs = (\n            Aggregator.max_expr(df) + Aggregator.mean_expr(df) + Aggregator.var_expr(df)\n        )\n        return exprs\n            ","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:09:25.976130Z","iopub.execute_input":"2024-06-07T08:09:25.976583Z","iopub.status.idle":"2024-06-07T08:09:25.995044Z","shell.execute_reply.started":"2024-06-07T08:09:25.976540Z","shell.execute_reply":"2024-06-07T08:09:25.993578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Utility:\n    \n    @staticmethod\n    def get_feat_defs(endingwith : str) -> None:\n        \n        feat_defs : pl.DataFrame = pl.read_csv(ROOT / \"feature_definitions.csv\")\n        \n        filtered_feats : pl.DataFrame = feat_defs.filter(\n            pl.col(\"Variable\").apply(lambda var : var.endswith(ending_with))\n            )\n        \n        with pl.Config(fmt_str_lengths = 200, tbl_rows = -1):\n            print(filtered_feats)\n        \n        filtered_feats = None\n        feat_defs = None\n    \n    @staticmethod\n    def find_index(lst : list[Any], item : Any) -> int | None : \n        try : \n            return lst.index(item)\n        except ValueError:\n            return None\n    \n    @staticmethod\n    def dtype_to_str(dtype : pl.DataType) -> str:\n        \n        dtype_map = {\n            pl.Decimal : \"Decimal\",\n            pl.Float32 : \"Float32\",\n            pl.Float64 : \"Float64\",\n            pl.UInt8: \"UInt8\",\n            pl.UInt16: \"UInt16\",\n            pl.UInt32: \"UInt32\",\n            pl.UInt64: \"UInt64\",\n            pl.Int8: \"Int8\",\n            pl.Int16: \"Int16\",\n            pl.Int32: \"Int32\",\n            pl.Int64: \"Int64\",\n            pl.Date: \"Date\",\n            pl.Datetime: \"Datetime\",\n            pl.Duration: \"Duration\",\n            pl.Time: \"Time\",\n            pl.Array: \"Array\",\n            pl.List: \"List\",\n            pl.Struct: \"Struct\",\n            pl.String: \"String\",\n            pl.Categorical: \"Categorical\",\n            pl.Enum: \"Enum\",\n            pl.Utf8: \"Utf8\",\n            pl.Binary: \"Binary\",\n            pl.Boolean: \"Boolean\",\n            pl.Null: \"Null\",\n            pl.Object: \"Object\",\n            pl.Unknown: \"Unknown\",\n        }\n        \n        return dtype_map.get(dtype)\n    \n    @staticmethod\n    def find_feat_occur(regex_path : str, ending_with : str) -> pl.DataFrame:\n        \n        feat_defs : pl.DataFrame = pl.read_csv(ROOT / \"feature_definitions.csv\").filter(\n            pl.col(\"Variable\").apply(lambda var : var.endswith(ending_with))\n        )\n        \n        feat_defs.sort(by = [\"Variable\"])\n        \n        feats : list[pl.String] = feat_defs[\"Variable\"].to_list()\n        feats.sort()\n        \n        occurences : list[list] = [[set(), set()] for _ in range(feat_defs.height)]\n            \n        for path in glob(str(regex_path)):\n            df_schema : dict = pl.read_parquet_schema(path)\n            \n            for feat, dtype in df_schema.items():\n                index : int = Utility.find_index(feats, feat)\n                if index != None:\n                    occurences[index][0].add(Utility.dtype_to_str(dtype))\n                    occurences[index][1].add(Path(path).stem)\n            \n        data_types : list[str] = [None] * feat_defs.height \n        file_locs : list[str] = [None] * feat_defs.height\n        \n        for i, feat in enumerate(feats):\n            data_types[i] = list(occurences[i][0])\n            file_locs[i] = list(occurrences[i][1])\n        \n        feat_defs = feat_defs.with_columns(pl.Series(data_types).alias(\"Data_Type(s)\"))\n        feat_defs = feat_defs.with_column(pl.series(data_types).alias(\"File_Loc(s)\"))\n        \n        return feat_defs\n    \n    def reduce_memory_usage(df : pl.DataFrame, name) -> pl.DataFrame :\n        int_types = [\n            pl.Int32,\n            pl.Int16,\n            pl.Int32,\n            pl.Int64,\n            pl.UInt8,\n            pl.UInt16,\n            pl.UInt32,\n            pl.UInt64,\n        ]\n        float_types = [pl.Float32, pl.Float64]\n        \n        for col in df.columns:\n            col_type = df[col].dtype\n            if col_type in int_types + float_types:\n                c_min = df[col].min()\n                c_max = df[col].max()\n                \n                if c_min is not None and c_max is not None : \n                    if col_type in int_types :\n                        if c_min >= 0 :\n                            if (\n                                c_min >= np.iinfo(np.uint8).min \n                                and c_max <= np.iinfo(np.uint8).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int8))\n                            elif (\n                                c_min >= np.iinfo(np.uint16).min \n                                and c_max <= np.iinfo(np.uint16).max \n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt16))\n                            elif (\n                                c_min >= np.iinfo(np.uint32).min \n                                and c_max <= np.iinfo(np.uint32).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt32))\n                            elif (\n                                c_min >= np.iinfo(np.uint64).min\n                                and c_max <= np.iinfo(np.uint64).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt64))\n                        else:\n                            if (\n                                c_min >= np.iinfo(np.int8).min\n                                and c_max <= np.iinfo(np.int8).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int8))\n                            elif (\n                                c_min >= np.iinfo(np.int16).min\n                                and c_max <= np.iinfo(np.int16).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int16))\n                            elif (\n                                c_min >= np.iinfo(np.int32).min\n                                and c_max <= np.iinfo(np.int32).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int32))\n                            elif (\n                                c_min >= np.iinfo(np.int64).min\n                                and c_max <= np.iinfo(np.int64).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int64))\n                    elif col_type in float_types:\n                        if (\n                            c_min > np.finfo(np.float32).min\n                            and c_max < np.finfo(np.float32).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.Float32))\n\n        print(\n            f\"Memory usage of dataframe \\\"{name}\\\" became {round(df.estimated_size('mb'), 4)} MB.\"\n        )\n\n        return df\n    \n    def to_pandas(df: pl.DataFrame, cat_cols : list[str] = None) -> (pd.DataFrame, list[str]) :\n        df : pd.DataFrame = df.to_pandas()\n        \n        if cat_cols is None:\n            cat_cols = list(df.select_dtypes(\"object\").columns)\n        \n        df[cat_cols] = df[cat_cols].astype(\"str\")\n        \n        return df, cat_cols\n                                \n                            ","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:09:26.449313Z","iopub.execute_input":"2024-06-07T08:09:26.449739Z","iopub.status.idle":"2024-06-07T08:09:26.491288Z","shell.execute_reply.started":"2024-06-07T08:09:26.449703Z","shell.execute_reply":"2024-06-07T08:09:26.489798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_cols(df : pl.DataFrame) -> pl.DataFrame:\n    \n    for col in df.columns:\n        if col not in [\"case_id\", \"year\", \"month\", \"week_num\", \"target\"]:\n            null_pct = df[col].is_null().mean()\n            \n            if null_pct > 0.95:\n                df = df.drop(col)\n    \n    for col in df.columns:\n        if (col not in [\"case_id\", \"year\", \"month\", \"week_num\", \"target\"]) & (df[col].dtype == pl.String):\n            freq = df[col].n_unique()\n            if (freq > 200) | (freq == 1):\n                df = df.drop(col)\n    \n    return df\n\ndef transform_cols(df: pl.DataFrame) -> pl.DataFrame:\n    if \"riskassesment_302T\" in df.columns:\n        if df[\"riskassesment_302T\"].dtype == pl.Null:\n            df = df.with_columns(\n                [\n                    pl.Series(\n                        \"riskassesment_302T_rng\", df[\"riskassesment_302T\"], pl.UInt8\n                    ),\n                    pl.Series(\n                        \"riskassesment_302T_mean\", df[\"riskassesment_302T\"], pl.UInt8\n                    ),\n                ]\n            )\n        else:\n            pct_low: pl.Series = (\n                df[\"riskassesment_302T\"]\n                .str.split(\" - \")\n                .apply(lambda x: x[0].replace(\"%\", \"\"))\n                .cast(pl.UInt8)\n            )\n            pct_high: pl.Series = (\n                df[\"riskassesment_302T\"]\n                .str.split(\" - \")\n                .apply(lambda x: x[1].replace(\"%\", \"\"))\n                .cast(pl.UInt8)\n            )\n\n            diff: pl.Series = pct_high - pct_low\n            avg: pl.Series = ((pct_low + pct_high) / 2).cast(pl.Float32)\n\n            del pct_high, pct_low\n            gc.collect()\n\n            df = df.with_columns(\n                [\n                    diff.alias(\"riskassesment_302T_rng\"),\n                    avg.alias(\"riskassesment_302T_mean\"),\n                ]\n            )\n\n        df.drop(\"riskassesment_302T\")\n\n    return df\n\ndef handle_dates(df : pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if col.endswith(\"D\"):\n            df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n            df = df.with_columns(pl.col(col).dt.total_days().cast(pl.Int32))\n    \n    df = df.rename({\n        \"MONTH\" : \"month\",\n        \"WEEK_NUM\" : \"week_num\"\n    })\n    \n    df = df.with_columns(\n        [\n            pl.col(\"date_decision\").dt.year().alias('year').cast(pl.Int16),\n            pl.col(\"date_decision\").dt.day().alias(\"day\").cast(pl.UInt8),\n        ]\n    )\n    \n    return df.drop(\"date_decision\")","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:09:26.741361Z","iopub.execute_input":"2024-06-07T08:09:26.741876Z","iopub.status.idle":"2024-06-07T08:09:26.761387Z","shell.execute_reply.started":"2024-06-07T08:09:26.741840Z","shell.execute_reply":"2024-06-07T08:09:26.760099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store: dict = {\n    \"df_base\": SchemaGen.scan_files(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        SchemaGen.scan_files(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        SchemaGen.scan_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        SchemaGen.scan_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n        SchemaGen.scan_files(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ],\n}","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:09:27.009417Z","iopub.execute_input":"2024-06-07T08:09:27.010538Z","iopub.status.idle":"2024-06-07T08:09:31.109615Z","shell.execute_reply.started":"2024-06-07T08:09:27.010492Z","shell.execute_reply":"2024-06-07T08:09:31.108254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndf_train: pl.LazyFrame = (\n    SchemaGen.join_dataframes(**data_store)\n    .pipe(filter_cols)\n    .pipe(handle_dates)\n    .pipe(Utility.reduce_memory_usage, \"df_train\")\n)\ndel data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:09:31.111540Z","iopub.execute_input":"2024-06-07T08:09:31.111929Z","iopub.status.idle":"2024-06-07T08:12:50.979473Z","shell.execute_reply.started":"2024-06-07T08:09:31.111898Z","shell.execute_reply":"2024-06-07T08:12:50.978247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store: dict = {\n    \"df_base\": SchemaGen.scan_files(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        SchemaGen.scan_files(TEST_DIR / \"test_static_cb_0.parquet\"),\n        SchemaGen.scan_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        SchemaGen.scan_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_other_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_person_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        SchemaGen.scan_files(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        SchemaGen.scan_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n        SchemaGen.scan_files(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n    ],\n}\n\ndf_test: pl.DataFrame = (\n    SchemaGen.join_dataframes(**data_store)\n    .pipe(handle_dates)\n    .select([col for col in df_train.columns if col != \"target\"])\n    .pipe(Utility.reduce_memory_usage, \"df_test\")\n)\n\ndel data_store\ngc.collect()\n\nprint(f\"Test data shape: {df_test.shape}\")\n\ndf_test.write_parquet(\"test_final.parquet\", compression=\"lz4\")","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:10:59.690307Z","iopub.execute_input":"2024-06-07T09:10:59.690851Z","iopub.status.idle":"2024-06-07T09:11:07.442503Z","shell.execute_reply.started":"2024-06-07T09:10:59.690809Z","shell.execute_reply":"2024-06-07T09:11:07.441121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, cat_cols = Utility.to_pandas(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:12:50.982015Z","iopub.execute_input":"2024-06-07T08:12:50.982450Z","iopub.status.idle":"2024-06-07T08:13:17.924351Z","shell.execute_reply.started":"2024-06-07T08:12:50.982409Z","shell.execute_reply":"2024-06-07T08:13:17.922901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_test, cat_cols = Utility.to_pandas(df_test, cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:11:12.508424Z","iopub.execute_input":"2024-06-07T09:11:12.508871Z","iopub.status.idle":"2024-06-07T09:11:12.548876Z","shell.execute_reply.started":"2024-06-07T09:11:12.508836Z","shell.execute_reply":"2024-06-07T09:11:12.547443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_col = df_train[[col for col in df_train.columns if col not in cat_cols]]","metadata":{"execution":{"iopub.status.busy":"2024-06-03T09:54:48.992770Z","iopub.execute_input":"2024-06-03T09:54:48.993273Z","iopub.status.idle":"2024-06-03T09:54:49.003183Z","shell.execute_reply.started":"2024-06-03T09:54:48.993235Z","shell.execute_reply":"2024-06-03T09:54:49.001776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-06-05T08:55:29.314074Z","iopub.execute_input":"2024-06-05T08:55:29.314609Z","iopub.status.idle":"2024-06-05T08:55:29.329333Z","shell.execute_reply.started":"2024-06-05T08:55:29.314566Z","shell.execute_reply":"2024-06-05T08:55:29.327841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"max_incometype_1044T\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-06-05T07:46:18.169688Z","iopub.execute_input":"2024-06-05T07:46:18.170170Z","iopub.status.idle":"2024-06-05T07:46:18.360306Z","shell.execute_reply.started":"2024-06-05T07:46:18.170137Z","shell.execute_reply":"2024-06-05T07:46:18.358460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nplt.figure(figsize = (10, 6))\nax = sns.countplot(data = df_train, x = \"max_incometype_1044T\", hue = \"target\", palette=\"pastel\", dodge = True)\nhandles, labels = ax.get_legend_handles_labels()\nnew_labels = ['No' if label == '0' else 'Yes' for label in labels]\nax.legend(handles, new_labels, title=\"Default on Loans\")\nplt.title(\"Comparision of Income type by credit default\")\nplt.xlabel(\"Income Type\")\nplt.xticks(rotation=-45)\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-05T08:08:55.703982Z","iopub.execute_input":"2024-06-05T08:08:55.704467Z","iopub.status.idle":"2024-06-05T08:08:58.154198Z","shell.execute_reply.started":"2024-06-05T08:08:55.704430Z","shell.execute_reply":"2024-06-05T08:08:58.152297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"income_percent = df_train.groupby([\"max_incometype_1044T\", \"target\"]).size().groupby(level = 0).apply(\nlambda x : 100 * x / float(x.sum())).unstack().fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-06-05T08:41:44.809578Z","iopub.execute_input":"2024-06-05T08:41:44.810110Z","iopub.status.idle":"2024-06-05T08:41:45.093086Z","shell.execute_reply.started":"2024-06-05T08:41:44.810065Z","shell.execute_reply":"2024-06-05T08:41:45.091792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"income_percent","metadata":{"execution":{"iopub.status.busy":"2024-06-05T08:41:47.910897Z","iopub.execute_input":"2024-06-05T08:41:47.911417Z","iopub.status.idle":"2024-06-05T08:41:47.927595Z","shell.execute_reply.started":"2024-06-05T08:41:47.911377Z","shell.execute_reply":"2024-06-05T08:41:47.926086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = income_percent.plot(kind='bar', stacked=True, color=['green', 'red'], figsize=(10, 6))\nhandles, labels = ax.get_legend_handles_labels()\nnew_labels = ['No' if label == '0' else 'Yes' for label in labels]\nax.legend(handles, new_labels, title=\"Default on Loans\")\nplt.title('Percentage of Loan Default and Paid by Income Type')\nplt.xlabel('Income Type')\nplt.ylabel('Percentage')\nplt.xticks(rotation=45)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-05T08:54:05.693319Z","iopub.execute_input":"2024-06-05T08:54:05.693765Z","iopub.status.idle":"2024-06-05T08:54:06.148059Z","shell.execute_reply.started":"2024-06-05T08:54:05.693731Z","shell.execute_reply":"2024-06-05T08:54:06.146376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"max_housetype_905L\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-06-05T08:58:48.489779Z","iopub.execute_input":"2024-06-05T08:58:48.491262Z","iopub.status.idle":"2024-06-05T08:58:48.656407Z","shell.execute_reply.started":"2024-06-05T08:58:48.491204Z","shell.execute_reply":"2024-06-05T08:58:48.654902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize = (10, 6))\nax = sns.countplot(data = df_train, x = \"max_housetype_905L\", hue = \"target\", palette = \"pastel\", dodge = True)\nhandles, labels = ax.get_legend_handles_labels()\nnew_labels = ['No' if label == '0' else 'Yes' for label in labels]\nax.legend(handles, new_labels, title=\"Default on Loans\")\nplt.title(\"Comparision of house type by credit default\")\nplt.xlabel(\"House Type\")\nplt.xticks(rotation=-45)\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-05T09:11:00.437185Z","iopub.execute_input":"2024-06-05T09:11:00.437738Z","iopub.status.idle":"2024-06-05T09:11:02.834080Z","shell.execute_reply.started":"2024-06-05T09:11:00.437696Z","shell.execute_reply":"2024-06-05T09:11:02.832911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"bankacctype_710L\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-06-06T09:29:44.055392Z","iopub.execute_input":"2024-06-06T09:29:44.055860Z","iopub.status.idle":"2024-06-06T09:29:44.088514Z","shell.execute_reply.started":"2024-06-06T09:29:44.055826Z","shell.execute_reply":"2024-06-06T09:29:44.087391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndf_train[\"maxdbddpdtollast6m_4187119P\"].dropna().plot(kind = 'hist')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-06T09:50:58.614997Z","iopub.execute_input":"2024-06-06T09:50:58.615434Z","iopub.status.idle":"2024-06-06T09:50:58.962847Z","shell.execute_reply.started":"2024-06-06T09:50:58.615378Z","shell.execute_reply":"2024-06-06T09:50:58.961517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"due_date_outlier = df_train[df_train['maxdbddpdtollast6m_4187119P'] > 1000][\"target\"]\n","metadata":{"execution":{"iopub.status.busy":"2024-06-06T10:03:40.517845Z","iopub.execute_input":"2024-06-06T10:03:40.519128Z","iopub.status.idle":"2024-06-06T10:03:40.941153Z","shell.execute_reply.started":"2024-06-06T10:03:40.519083Z","shell.execute_reply":"2024-06-06T10:03:40.939845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"due_date_outlier.value_counts()/ due_date_outlier.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-06-06T10:06:20.689425Z","iopub.execute_input":"2024-06-06T10:06:20.689866Z","iopub.status.idle":"2024-06-06T10:06:20.701921Z","shell.execute_reply.started":"2024-06-06T10:06:20.689830Z","shell.execute_reply":"2024-06-06T10:06:20.700384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"before_due_date = df_train[df_train['maxdbddpdtollast6m_4187119P'] < 0][\"target\"]\nbefore_due_date.value_counts() / before_due_date.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-06-06T10:06:32.308911Z","iopub.execute_input":"2024-06-06T10:06:32.309292Z","iopub.status.idle":"2024-06-06T10:06:33.648288Z","shell.execute_reply.started":"2024-06-06T10:06:32.309264Z","shell.execute_reply":"2024-06-06T10:06:33.647089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"after_due_date = df_train[ (df_train['maxdbddpdtollast6m_4187119P'] > 0) & (df_train['maxdbddpdtollast6m_4187119P'] < 1000) ][\"target\"]\nafter_due_date.value_counts()/ after_due_date.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-06-06T10:08:04.151628Z","iopub.execute_input":"2024-06-06T10:08:04.152955Z","iopub.status.idle":"2024-06-06T10:08:05.416761Z","shell.execute_reply.started":"2024-06-06T10:08:04.152897Z","shell.execute_reply":"2024-06-06T10:08:05.415273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(\"target\").agg(\"currdebt_22A\").mean()","metadata":{"execution":{"iopub.status.busy":"2024-06-06T10:37:00.457789Z","iopub.execute_input":"2024-06-06T10:37:00.459506Z","iopub.status.idle":"2024-06-06T10:37:00.502858Z","shell.execute_reply.started":"2024-06-06T10:37:00.459457Z","shell.execute_reply":"2024-06-06T10:37:00.501494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(\"target\").agg(\"currdebt_22A\").std()","metadata":{"execution":{"iopub.status.busy":"2024-06-06T10:37:33.266661Z","iopub.execute_input":"2024-06-06T10:37:33.267167Z","iopub.status.idle":"2024-06-06T10:37:33.313995Z","shell.execute_reply.started":"2024-06-06T10:37:33.267135Z","shell.execute_reply":"2024-06-06T10:37:33.312636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(data = df_train, x = \"target\", y = \"currdebt_22A\", palette = \"Set2\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Filter out the outliers\nfiltered_df = df_train[(df_train['currdebt_22A'] > 0)]\n\n# Create the box plot with the filtered data\nplt.figure(figsize=(10, 6))\nsns.boxplot(data=filtered_df, x='target', y='currdebt_22A', palette=\"Set2\")\n\n# Customize the plot\nplt.title('Box Plot of currdebt_22A with Hue by Target (Outliers Removed)')\nplt.xlabel('Target')\nplt.ylabel('currdebt_22A')\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-06T11:10:45.536891Z","iopub.execute_input":"2024-06-06T11:10:45.538194Z","iopub.status.idle":"2024-06-06T11:10:47.821055Z","shell.execute_reply.started":"2024-06-06T11:10:45.538152Z","shell.execute_reply":"2024-06-06T11:10:47.819871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:37:46.429620Z","iopub.execute_input":"2024-06-07T08:37:46.430041Z","iopub.status.idle":"2024-06-07T08:37:46.654762Z","shell.execute_reply.started":"2024-06-07T08:37:46.430010Z","shell.execute_reply":"2024-06-07T08:37:46.653499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = df_train, x = \"education_88M\", hue = \"target\", palette=\"pastel\", dodge = True)","metadata":{"execution":{"iopub.status.busy":"2024-06-07T08:39:03.088090Z","iopub.execute_input":"2024-06-07T08:39:03.089286Z","iopub.status.idle":"2024-06-07T08:39:05.434892Z","shell.execute_reply.started":"2024-06-07T08:39:03.089239Z","shell.execute_reply":"2024-06-07T08:39:05.433634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_columns = df_train.select_dtypes(include = [\"number\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:14:04.724298Z","iopub.execute_input":"2024-06-07T09:14:04.724721Z","iopub.status.idle":"2024-06-07T09:14:04.933068Z","shell.execute_reply.started":"2024-06-07T09:14:04.724686Z","shell.execute_reply":"2024-06-07T09:14:04.931670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlation = numeric_columns.corr()","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:16:41.680570Z","iopub.execute_input":"2024-06-07T09:16:41.681174Z","iopub.status.idle":"2024-06-07T09:17:06.883994Z","shell.execute_reply.started":"2024-06-07T09:16:41.681103Z","shell.execute_reply":"2024-06-07T09:17:06.882908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_correlation_columns = correlation[\"target\"].sort_values(ascending=False).head(10).index","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:27:33.264328Z","iopub.execute_input":"2024-06-07T09:27:33.264760Z","iopub.status.idle":"2024-06-07T09:27:33.272603Z","shell.execute_reply.started":"2024-06-07T09:27:33.264723Z","shell.execute_reply":"2024-06-07T09:27:33.271280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_10_correlation = df_train[top_correlation_columns].corr()","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:28:05.373813Z","iopub.execute_input":"2024-06-07T09:28:05.374294Z","iopub.status.idle":"2024-06-07T09:28:05.405695Z","shell.execute_reply.started":"2024-06-07T09:28:05.374258Z","shell.execute_reply":"2024-06-07T09:28:05.404407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 10))\nsns.heatmap(top_10_correlation, annot = True, cmap='coolwarm', linewidths=0.5)\nplt.title(\"loan default correlation map\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:28:49.666308Z","iopub.execute_input":"2024-06-07T09:28:49.666772Z","iopub.status.idle":"2024-06-07T09:28:50.416053Z","shell.execute_reply.started":"2024-06-07T09:28:49.666736Z","shell.execute_reply":"2024-06-07T09:28:50.414723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VotingModel(BaseEstimator, ClassifierMixin):\n    \n    def __init__(self, estimators : list[BaseEstimator]):\n        super().__init__()\n        self.estimators = estimators\n    \n    def fit(self, X, y = None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis = 0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis = 0)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:42:01.303322Z","iopub.execute_input":"2024-06-07T09:42:01.304236Z","iopub.status.idle":"2024-06-07T09:42:01.313467Z","shell.execute_reply.started":"2024-06-07T09:42:01.304191Z","shell.execute_reply":"2024-06-07T09:42:01.311902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm: pd.DataFrame = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndevice: str = \"gpu\"\nest_cnt: int = 6000\n\nDRY_RUN = True if df_subm.shape[0] == 10 else False\nif DRY_RUN:\n    device = \"cpu\"\n    df_train = df_train.iloc[:50000]\n    est_cnt: int = 600\n\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:42:05.142417Z","iopub.execute_input":"2024-06-07T09:42:05.142816Z","iopub.status.idle":"2024-06-07T09:42:05.158086Z","shell.execute_reply.started":"2024-06-07T09:42:05.142787Z","shell.execute_reply":"2024-06-07T09:42:05.156381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df_train.drop(columns = [\"target\", \"case_id\", \"week_num\"])\ny = df_train[\"target\"]\n\nweeks = df_train[\"week_num\"]\n\ndel df_train\n\ngc.collect()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:42:06.728837Z","iopub.execute_input":"2024-06-07T09:42:06.729263Z","iopub.status.idle":"2024-06-07T09:42:10.701710Z","shell.execute_reply.started":"2024-06-07T09:42:06.729233Z","shell.execute_reply":"2024-06-07T09:42:10.700366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits = 5, shuffle = False)\n\nparams1 = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bynode\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"device\": device,\n    \"extra_trees\": True,\n    \"learning_rate\": 0.05,\n    \"l1_regularization\": 0.1,\n    \"l2_regularization\": 10,\n    \"max_depth\": 20,\n    \"metric\": \"auc\",\n    \"n_estimators\": 2000,\n    \"num_leaves\": 64,\n    \"objective\": \"binary\",\n    \"random_state\": 42,\n    \"verbose\": -1,\n}\n\nparams2 = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bynode\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"device\": device,\n    \"extra_trees\": True,\n    \"learning_rate\": 0.03,\n    \"l1_regularization\": 0.1,\n    \"l2_regularization\": 10,\n    \"max_depth\": 16,\n    \"metric\": \"auc\",\n    \"n_estimators\": 2000,\n    \"num_leaves\": 72,\n    \"objective\": \"binary\",\n    \"random_state\": 42,\n    \"verbose\": -1,\n}\n\nfitted_models_cat = []\nfitted_models_lgb = []\n\ncv_scores_cat = []\ncv_scores_lgb = []\n\nfitted_models_cat = []\nfitted_models_lgb = []\n\ncv_scores_cat = []\ncv_scores_lgb = []\n\niter_cnt = 0\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    train_pool = Pool(X_train, y_train, cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid, cat_features=cat_cols)\n\n    clf = CatBoostClassifier(\n        best_model_min_trees = 1200,\n        boosting_type = \"Plain\",\n        eval_metric = \"AUC\",\n        iterations = est_cnt,\n        learning_rate = 0.05,\n        l2_leaf_reg = 10,\n        max_leaves = 64,\n        random_seed = 42,\n        task_type = \"CPU\",\n        use_best_model = True\n    )\n\n    clf.fit(train_pool, eval_set=val_pool, verbose=False)\n    fitted_models_cat.append(clf)\n\n    y_pred_valid = clf.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n\n    X_train[cat_cols] = X_train[cat_cols].astype(\"category\")\n    X_valid[cat_cols] = X_valid[cat_cols].astype(\"category\")\n\n    if iter_cnt % 2 == 0:\n        model = lgb.LGBMClassifier(**params1)\n    else:\n        model = lgb.LGBMClassifier(**params2)\n\n    model.fit(\n        X_train,\n        y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(100)],\n    )\n    fitted_models_lgb.append(model)\n\n    y_pred_valid = model.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_lgb.append(auc_score)\n\n    iter_cnt += 1\n\nmodel = VotingModel(fitted_models_cat + fitted_models_lgb)\n\nprint(f\"\\nCV AUC scores for CatBoost: {cv_scores_cat}\")\nprint(f\"Maximum CV AUC score for Catboost: {max(cv_scores_cat)}\", end=\"\\n\\n\")\n\n\nprint(f\"CV AUC scores for LGBM: {cv_scores_lgb}\")\nprint(f\"Maximum CV AUC score for LGBM: {max(cv_scores_lgb)}\", end=\"\\n\\n\")\n\ndel X, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-07T09:42:48.073227Z","iopub.execute_input":"2024-06-07T09:42:48.074359Z","iopub.status.idle":"2024-06-07T10:10:59.796438Z","shell.execute_reply.started":"2024-06-07T09:42:48.074315Z","shell.execute_reply":"2024-06-07T10:10:59.795018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test: pd.DataFrame = df_test.drop(columns=[\"week_num\"]).set_index(\"case_id\")\n\nX_test[cat_cols] = X_test[cat_cols].astype(\"category\")\n\ny_pred: pd.Series = pd.Series(model.predict_proba(X_test)[:, 1], index=X_test.index)\n\ndf_subm[\"score\"] = y_pred\n\ndisplay(df_subm)\n\ndf_subm.to_csv(\"submission.csv\")\n\ndel X_test, y_pred, df_subm\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-07T10:10:59.798631Z","iopub.execute_input":"2024-06-07T10:10:59.799005Z","iopub.status.idle":"2024-06-07T10:11:00.538536Z","shell.execute_reply.started":"2024-06-07T10:10:59.798974Z","shell.execute_reply":"2024-06-07T10:11:00.537237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}