{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 100-HomeCredit Notebook","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom glob import glob\nimport gc\nfrom datetime import datetime\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.metrics import roc_auc_score \nfrom typing import Any\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:48:24.188701Z","iopub.execute_input":"2024-05-19T04:48:24.189350Z","iopub.status.idle":"2024-05-19T04:48:27.445300Z","shell.execute_reply.started":"2024-05-19T04:48:24.189310Z","shell.execute_reply":"2024-05-19T04:48:27.444047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility Class that creates multiple function that helped process data","metadata":{}},{"cell_type":"code","source":"class Utility:\n    @staticmethod\n    def get_feat_defs(ending_with: str) -> None:\n        \"\"\"\n        Retrieves feature definitions from a CSV file based on the specified ending.\n\n        Args:\n        - ending_with (str): Ending to filter feature definitions.\n\n        Returns:\n        - pl.DataFrame: Filtered feature definitions.\n        \"\"\"\n        feat_defs: pl.DataFrame = pl.read_csv(dataPath / \"feature_definitions.csv\")\n\n        filtered_feats: pl.DataFrame = feat_defs.filter(\n            pl.col(\"Variable\").apply(lambda var: var.endswith(ending_with))\n        )\n\n        with pl.Config(fmt_str_lengths=200, tbl_rows=-1):\n            print(filtered_feats)\n\n        filtered_feats = None\n        feat_defs = None\n\n    @staticmethod\n    def find_index(lst: list[Any], item: Any) -> int | None:\n        \"\"\"\n        Finds the index of an item in a list.\n\n        Args:\n        - lst (list): List to search.\n        - item (Any): Item to find in the list.\n\n        Returns:\n        - int | None: Index of the item if found, otherwise None.\n        \"\"\"\n        try:\n            return lst.index(item)\n        except ValueError:\n            return None\n\n    @staticmethod\n    def dtype_to_str(dtype: pl.DataType) -> str:\n        \"\"\"\n        Converts Polars data type to string representation.\n\n        Args:\n        - dtype (pl.DataType): Polars data type.\n\n        Returns:\n        - str: String representation of the data type.\n        \"\"\"\n        dtype_map = {\n            pl.Decimal: \"Decimal\",\n            pl.Float32: \"Float32\",\n            pl.Float64: \"Float64\",\n            pl.UInt8: \"UInt8\",\n            pl.UInt16: \"UInt16\",\n            pl.UInt32: \"UInt32\",\n            pl.UInt64: \"UInt64\",\n            pl.Int8: \"Int8\",\n            pl.Int16: \"Int16\",\n            pl.Int32: \"Int32\",\n            pl.Int64: \"Int64\",\n            pl.Date: \"Date\",\n            pl.Datetime: \"Datetime\",\n            pl.Duration: \"Duration\",\n            pl.Time: \"Time\",\n            pl.Array: \"Array\",\n            pl.List: \"List\",\n            pl.Struct: \"Struct\",\n            pl.String: \"String\",\n            pl.Categorical: \"Categorical\",\n            pl.Enum: \"Enum\",\n            pl.Utf8: \"Utf8\",\n            pl.Binary: \"Binary\",\n            pl.Boolean: \"Boolean\",\n            pl.Null: \"Null\",\n            pl.Object: \"Object\",\n            pl.Unknown: \"Unknown\",\n        }\n\n        return dtype_map.get(dtype)\n\n    @staticmethod\n    def find_feat_occur(regex_path: str, ending_with: str) -> pl.DataFrame:\n        \"\"\"\n        Finds occurrences of features ending with a specific string in Parquet files.\n\n        Args:\n        - regex_path (str): Regular expression to match Parquet file paths.\n        - ending_with (str): Ending to filter feature names.\n\n        Returns:\n        - pl.DataFrame: DataFrame containing feature definitions, data types, and file locations.\n        \"\"\"\n        feat_defs: pl.DataFrame = pl.read_csv(dataPath / \"feature_definitions.csv\").filter(\n            pl.col(\"Variable\").apply(lambda var: var.endswith(ending_with))\n        )\n        feat_defs.sort(by=[\"Variable\"])\n\n        feats: list[pl.String] = feat_defs[\"Variable\"].to_list()\n        feats.sort()\n\n        occurrences: list[list] = [[set(), set()] for _ in range(feat_defs.height)]\n\n        for path in glob(str(regex_path)):\n            df_schema: dict = pl.read_parquet_schema(path)\n\n            for feat, dtype in df_schema.items():\n                index: int = Utility.find_index(feats, feat)\n                if index != None:\n                    occurrences[index][0].add(Utility.dtype_to_str(dtype))\n                    occurrences[index][1].add(Path(path).stem)\n\n        data_types: list[str] = [None] * feat_defs.height\n        file_locs: list[str] = [None] * feat_defs.height\n\n        for i, feat in enumerate(feats):\n            data_types[i] = list(occurrences[i][0])\n            file_locs[i] = list(occurrences[i][1])\n\n        feat_defs = feat_defs.with_columns(pl.Series(data_types).alias(\"Data_Type(s)\"))\n        feat_defs = feat_defs.with_columns(pl.Series(file_locs).alias(\"File_Loc(s)\"))\n\n        return feat_defs\n\n    def reduce_memory_usage(df: pl.DataFrame, name) -> pl.DataFrame:\n        \"\"\"\n        Reduces memory usage of a DataFrame by converting column types.\n\n        Args:\n        - df (pl.DataFrame): DataFrame to optimize.\n        - name (str): Name of the DataFrame.\n\n        Returns:\n        - pl.DataFrame: Optimized DataFrame.\n        \"\"\"\n        print(\n            f\"Memory usage of dataframe \\\"{name}\\\" is {round(df.estimated_size('mb'), 4)} MB.\"\n        )\n\n        int_types = [\n            pl.Int8,\n            pl.Int16,\n            pl.Int32,\n            pl.Int64,\n            pl.UInt8,\n            pl.UInt16,\n            pl.UInt32,\n            pl.UInt64,\n        ]\n        float_types = [pl.Float32, pl.Float64]\n\n        for col in df.columns:\n            col_type = df[col].dtype\n            if col_type in int_types + float_types:\n                c_min = df[col].min()\n                c_max = df[col].max()\n\n                if c_min is not None and c_max is not None:\n                    if col_type in int_types:\n                        if c_min >= 0:\n                            if (\n                                c_min >= np.iinfo(np.uint8).min\n                                and c_max <= np.iinfo(np.uint8).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt8))\n                            elif (\n                                c_min >= np.iinfo(np.uint16).min\n                                and c_max <= np.iinfo(np.uint16).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt16))\n                            elif (\n                                c_min >= np.iinfo(np.uint32).min\n                                and c_max <= np.iinfo(np.uint32).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt32))\n                            elif (\n                                c_min >= np.iinfo(np.uint64).min\n                                and c_max <= np.iinfo(np.uint64).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.UInt64))\n                        else:\n                            if (\n                                c_min >= np.iinfo(np.int8).min\n                                and c_max <= np.iinfo(np.int8).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int8))\n                            elif (\n                                c_min >= np.iinfo(np.int16).min\n                                and c_max <= np.iinfo(np.int16).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int16))\n                            elif (\n                                c_min >= np.iinfo(np.int32).min\n                                and c_max <= np.iinfo(np.int32).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int32))\n                            elif (\n                                c_min >= np.iinfo(np.int64).min\n                                and c_max <= np.iinfo(np.int64).max\n                            ):\n                                df = df.with_columns(df[col].cast(pl.Int64))\n                    elif col_type in float_types:\n                        if (\n                            c_min > np.finfo(np.float32).min\n                            and c_max < np.finfo(np.float32).max\n                        ):\n                            df = df.with_columns(df[col].cast(pl.Float32))\n\n        print(\n            f\"Memory usage of dataframe \\\"{name}\\\" became {round(df.estimated_size('mb'), 4)} MB.\"\n        )\n\n        return df\n\n    def to_pandas(df: pl.DataFrame, cat_cols: list[str] = None) -> (pd.DataFrame, list[str]):  # type: ignore\n        \"\"\"\n        Converts a Polars DataFrame to a Pandas DataFrame.\n\n        Args:\n        - df (pl.DataFrame): Polars DataFrame to convert.\n        - cat_cols (list[str]): List of categorical columns. Default is None.\n\n        Returns:\n        - (pd.DataFrame, list[str]): Tuple containing the converted Pandas DataFrame and categorical columns.\n        \"\"\"\n        df: pd.DataFrame = df.to_pandas()\n\n        if cat_cols is None:\n            cat_cols = list(df.select_dtypes(\"object\").columns)\n\n        df[cat_cols] = df[cat_cols].astype(\"str\")\n\n        return df, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:48:27.448411Z","iopub.execute_input":"2024-05-19T04:48:27.449444Z","iopub.status.idle":"2024-05-19T04:48:27.492609Z","shell.execute_reply.started":"2024-05-19T04:48:27.449404Z","shell.execute_reply":"2024-05-19T04:48:27.491281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pipeline","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n\n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.95:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:48:27.494772Z","iopub.execute_input":"2024-05-19T04:48:27.495246Z","iopub.status.idle":"2024-05-19T04:48:27.511541Z","shell.execute_reply.started":"2024-05-19T04:48:27.495202Z","shell.execute_reply":"2024-05-19T04:48:27.510355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggergation Function","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:48:27.515302Z","iopub.execute_input":"2024-05-19T04:48:27.515832Z","iopub.status.idle":"2024-05-19T04:48:27.530144Z","shell.execute_reply.started":"2024-05-19T04:48:27.515791Z","shell.execute_reply":"2024-05-19T04:48:27.528820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        chunks.append(pl.read_parquet(path).pipe(Pipeline.set_table_dtypes))\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n    del chunks\n    gc.collect()\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:48:27.531487Z","iopub.execute_input":"2024-05-19T04:48:27.531870Z","iopub.status.idle":"2024-05-19T04:48:27.543268Z","shell.execute_reply.started":"2024-05-19T04:48:27.531840Z","shell.execute_reply":"2024-05-19T04:48:27.542019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### Preprocessing on Credit Bureau","metadata":{}},{"cell_type":"code","source":"train_credit_bureau_a_1 = read_files(dataPath + \"parquet_files/train/train_credit_bureau_a_1_*.parquet\", 1)\ntrain_credit_bureau_a_1 = train_credit_bureau_a_1.with_columns(\n    ((pl.col('max_dateofcredend_289D') - pl.col('max_dateofcredstart_739D')).dt.total_days()).alias('max_credit_duration_daysA')\n).with_columns(\n    ((pl.col('max_dateofcredend_353D') - pl.col('max_dateofcredstart_181D')).dt.total_days()).alias('max_closed_credit_duration_daysA')\n).with_columns(\n    ((pl.col('max_dateofrealrepmt_138D') - pl.col('max_overdueamountmax2date_1002D')).dt.total_days()).alias('max_time_from_overdue_to_closed_realrepmtA')\n).with_columns(\n    ((pl.col('max_dateofrealrepmt_138D') - pl.col('max_overdueamountmax2date_1142D')).dt.total_days()).alias('max_time_from_active_overdue_to_realrepmtA')\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:48:27.545096Z","iopub.execute_input":"2024-05-19T04:48:27.545615Z","iopub.status.idle":"2024-05-19T04:49:28.997295Z","shell.execute_reply.started":"2024-05-19T04:48:27.545568Z","shell.execute_reply":"2024-05-19T04:49:28.996028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_b_1 = read_file(dataPath + \"parquet_files/train/train_credit_bureau_b_1.parquet\", 1)\ntrain_credit_bureau_b_1 = train_credit_bureau_b_1.with_columns(\n    ((pl.col('max_contractmaturitydate_151D') - pl.col('max_contractdate_551D')).dt.total_days()).alias('contract_duration_days')\n).with_columns(\n    ((pl.col('max_lastupdate_260D') - pl.col('max_contractdate_551D')).dt.total_days()).alias('last_update_duration_days')\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:49:28.999068Z","iopub.execute_input":"2024-05-19T04:49:29.000607Z","iopub.status.idle":"2024-05-19T04:49:29.170441Z","shell.execute_reply.started":"2024-05-19T04:49:29.000558Z","shell.execute_reply":"2024-05-19T04:49:29.169148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing on Static","metadata":{}},{"cell_type":"code","source":"train_static = read_files(dataPath + \"parquet_files/train/train_static_0_*.parquet\")\ncondition_all_nan = (\n    pl.col('maxdbddpdlast1m_3658939P').is_null() &\n    pl.col('maxdbddpdtollast12m_3658940P').is_null() &\n    pl.col('maxdbddpdtollast6m_4187119P').is_null()\n)\n\ncondition_exceed_thresholds = (\n    (pl.col('maxdbddpdlast1m_3658939P') > 31) |\n    (pl.col('maxdbddpdtollast12m_3658940P') > 366) |\n    (pl.col('maxdbddpdtollast6m_4187119P') > 184)\n)\n\ntrain_static = train_static.with_columns(\n    pl.when(condition_all_nan | condition_exceed_thresholds)\n    .then(0)\n    .otherwise(1)\n    .alias('max_dbddpd_booleanP')\n)\n\ntrain_static = train_static.with_columns(\n    pl.when(\n        (pl.col('maxdbddpdlast1m_3658939P') <= 0) &\n        (pl.col('maxdbddpdtollast12m_3658940P') <= 0) &\n        (pl.col('maxdbddpdtollast6m_4187119P') <= 0)\n    )\n    .then(1)\n    .otherwise(0)\n    .alias('max_pays_debt_on_timeP')\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:49:29.172017Z","iopub.execute_input":"2024-05-19T04:49:29.172542Z","iopub.status.idle":"2024-05-19T04:49:35.140535Z","shell.execute_reply.started":"2024-05-19T04:49:29.172509Z","shell.execute_reply":"2024-05-19T04:49:35.139294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read train files\ndata_store = {\n    \"df_base\": read_file(dataPath + \"parquet_files/train/train_base.parquet\"),\n    \"depth_0\": [\n        read_file(dataPath + \"parquet_files/train/train_static_cb_0.parquet\"),\n        train_static,\n    ],\n    \"depth_1\": [\n        read_files(dataPath + \"parquet_files/train/train_applprev_1_*.parquet\", 1),\n        read_file(dataPath + \"parquet_files/train/train_tax_registry_a_1.parquet\", 1),\n        train_credit_bureau_b_1,\n        read_file(dataPath + \"parquet_files/train/train_tax_registry_c_1.parquet\", 1),\n        train_credit_bureau_a_1,\n        read_file(dataPath + \"parquet_files/train/train_credit_bureau_b_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/train/train_other_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/train/train_person_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/train/train_deposit_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/train/train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(dataPath + \"parquet_files/train/train_credit_bureau_b_2.parquet\", 2),\n    ]\n}\n","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:49:35.142345Z","iopub.execute_input":"2024-05-19T04:49:35.143232Z","iopub.status.idle":"2024-05-19T04:50:06.928029Z","shell.execute_reply.started":"2024-05-19T04:49:35.143185Z","shell.execute_reply":"2024-05-19T04:50:06.926527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature engineering\n\nIn this part, we can see a simple example of joining tables via `case_id`. Here the loading and joining is done with polars library. Polars library is blazingly fast and has much smaller memory footprint than pandas. ","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:06.932468Z","iopub.execute_input":"2024-05-19T04:50:06.932835Z","iopub.status.idle":"2024-05-19T04:50:06.939866Z","shell.execute_reply.started":"2024-05-19T04:50:06.932805Z","shell.execute_reply":"2024-05-19T04:50:06.939073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:06.941047Z","iopub.execute_input":"2024-05-19T04:50:06.942066Z","iopub.status.idle":"2024-05-19T04:50:06.956323Z","shell.execute_reply.started":"2024-05-19T04:50:06.942032Z","shell.execute_reply":"2024-05-19T04:50:06.955238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\ndf_train = df_train.pipe(Pipeline.handle_dates).pipe(Utility.reduce_memory_usage, \"df_train\")\nprint(\"train data shape:\\t\", df_train.shape)\n\ndel data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:06.958204Z","iopub.execute_input":"2024-05-19T04:50:06.959307Z","iopub.status.idle":"2024-05-19T04:50:28.844118Z","shell.execute_reply.started":"2024-05-19T04:50:06.959244Z","shell.execute_reply":"2024-05-19T04:50:28.842834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(df_train.head(10))","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:28.845592Z","iopub.execute_input":"2024-05-19T04:50:28.845968Z","iopub.status.idle":"2024-05-19T04:50:28.877048Z","shell.execute_reply.started":"2024-05-19T04:50:28.845934Z","shell.execute_reply":"2024-05-19T04:50:28.875730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1 = read_files(dataPath + \"parquet_files/test/test_credit_bureau_a_1_*.parquet\", 1)\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(\n    ((pl.col('max_dateofcredend_289D') - pl.col('max_dateofcredstart_739D')).dt.total_days()).alias('max_credit_duration_daysA')\n).with_columns(\n    ((pl.col('max_dateofcredend_353D') - pl.col('max_dateofcredstart_181D')).dt.total_days()).alias('max_closed_credit_duration_daysA')\n).with_columns(\n    ((pl.col('max_dateofrealrepmt_138D') - pl.col('max_overdueamountmax2date_1002D')).dt.total_days()).alias('max_time_from_overdue_to_closed_realrepmtA')\n).with_columns(\n    ((pl.col('max_dateofrealrepmt_138D') - pl.col('max_overdueamountmax2date_1142D')).dt.total_days()).alias('max_time_from_active_overdue_to_realrepmtA')\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:28.878774Z","iopub.execute_input":"2024-05-19T04:50:28.881697Z","iopub.status.idle":"2024-05-19T04:50:29.104204Z","shell.execute_reply.started":"2024-05-19T04:50:28.881607Z","shell.execute_reply":"2024-05-19T04:50:29.102988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_b_1 = read_file(dataPath + \"parquet_files/test/test_credit_bureau_b_1.parquet\", 1)\ntest_credit_bureau_b_1 = test_credit_bureau_b_1.with_columns(\n    ((pl.col('max_contractmaturitydate_151D') - pl.col('max_contractdate_551D')).dt.total_days()).alias('contract_duration_days')\n).with_columns(\n    ((pl.col('max_lastupdate_260D') - pl.col('max_contractdate_551D')).dt.total_days()).alias('last_update_duration_days')\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:29.105886Z","iopub.execute_input":"2024-05-19T04:50:29.106361Z","iopub.status.idle":"2024-05-19T04:50:29.126489Z","shell.execute_reply.started":"2024-05-19T04:50:29.106313Z","shell.execute_reply":"2024-05-19T04:50:29.125114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static = read_files(dataPath + \"parquet_files/test/test_static_0_*.parquet\")\ncondition_all_nan = (\n    pl.col('maxdbddpdlast1m_3658939P').is_null() &\n    pl.col('maxdbddpdtollast12m_3658940P').is_null() &\n    pl.col('maxdbddpdtollast6m_4187119P').is_null()\n)\n\ncondition_exceed_thresholds = (\n    (pl.col('maxdbddpdlast1m_3658939P') > 31) |\n    (pl.col('maxdbddpdtollast12m_3658940P') > 366) |\n    (pl.col('maxdbddpdtollast6m_4187119P') > 184)\n)\n\ntest_static = test_static.with_columns(\n    pl.when(condition_all_nan | condition_exceed_thresholds)\n    .then(0)\n    .otherwise(1)\n    .alias('max_dbddpd_booleanP')\n)\n\ntest_static = test_static.with_columns(\n    pl.when(\n        (pl.col('maxdbddpdlast1m_3658939P') <= 0) &\n        (pl.col('maxdbddpdtollast12m_3658940P') <= 0) &\n        (pl.col('maxdbddpdtollast6m_4187119P') <= 0)\n    )\n    .then(1)\n    .otherwise(0)\n    .alias('max_pays_debt_on_timeP')\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:29.128218Z","iopub.execute_input":"2024-05-19T04:50:29.129354Z","iopub.status.idle":"2024-05-19T04:50:29.303838Z","shell.execute_reply.started":"2024-05-19T04:50:29.129277Z","shell.execute_reply":"2024-05-19T04:50:29.302705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read test files\ndata_store = {\n    \"df_base\": read_file(dataPath + \"parquet_files/test/test_base.parquet\"),\n    \"depth_0\": [\n        read_file(dataPath + \"parquet_files/test/test_static_cb_0.parquet\"),\n        test_static,\n    ],\n    \"depth_1\": [\n        read_files(dataPath + \"parquet_files/test/test_applprev_1_*.parquet\", 1),\n        read_file(dataPath + \"parquet_files/test/test_tax_registry_a_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/test/test_tax_registry_b_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/test/test_tax_registry_c_1.parquet\", 1),\n        test_credit_bureau_a_1,\n        test_credit_bureau_b_1,\n        read_file(dataPath + \"parquet_files/test/test_other_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/test/test_person_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/test/test_deposit_1.parquet\", 1),\n        read_file(dataPath + \"parquet_files/test/test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(dataPath + \"parquet_files/test/test_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:29.305148Z","iopub.execute_input":"2024-05-19T04:50:29.305429Z","iopub.status.idle":"2024-05-19T04:50:29.529156Z","shell.execute_reply.started":"2024-05-19T04:50:29.305403Z","shell.execute_reply":"2024-05-19T04:50:29.527704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\ndf_test = df_test.pipe(Pipeline.handle_dates).pipe(Utility.reduce_memory_usage, \"df_train\")\nprint(\"test data shape:\\t\", df_test.shape)\n\ndel data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:29.530885Z","iopub.execute_input":"2024-05-19T04:50:29.531263Z","iopub.status.idle":"2024-05-19T04:50:29.794713Z","shell.execute_reply.started":"2024-05-19T04:50:29.531230Z","shell.execute_reply":"2024-05-19T04:50:29.793558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# filter out data\ndf_train = df_train.pipe(Pipeline.filter_cols)\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\n\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:29.796221Z","iopub.execute_input":"2024-05-19T04:50:29.796583Z","iopub.status.idle":"2024-05-19T04:50:33.133669Z","shell.execute_reply.started":"2024-05-19T04:50:29.796554Z","shell.execute_reply":"2024-05-19T04:50:33.132241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open('df_train_col.pkl', 'wb') as f:\n    pickle.dump(df_train.columns, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:33.135141Z","iopub.execute_input":"2024-05-19T04:50:33.135572Z","iopub.status.idle":"2024-05-19T04:50:33.142070Z","shell.execute_reply.started":"2024-05-19T04:50:33.135541Z","shell.execute_reply":"2024-05-19T04:50:33.140744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transfer to pandas\ndf_train, cat_cols = to_pandas(df_train)\n\nwith open('cat_cols.pkl', 'wb') as f:\n    pickle.dump(cat_cols, f)\n\ndf_test, cat_cols = to_pandas(df_test, cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T04:50:33.144151Z","iopub.execute_input":"2024-05-19T04:50:33.144570Z","iopub.status.idle":"2024-05-19T04:50:53.408214Z","shell.execute_reply.started":"2024-05-19T04:50:33.144529Z","shell.execute_reply":"2024-05-19T04:50:53.406850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nimport zipfile\nwith open('df_test.pkl', 'wb') as f:\n    pickle.dump(df_test, f)\n\n# with open('cat_cols.pkl', 'wb') as f:\n#     pickle.dump(cat_cols, f)\n\n# Compress the pickle files into a zip file\nwith zipfile.ZipFile('test_data.zip', 'w') as zipf:\n    zipf.write('df_test.pkl')\n    zipf.write('cat_cols.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Model","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n\n    def fit(self, X, y=None):\n        return self\n\n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n\n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = pd.read_csv(dataPath + \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\n# device: str = \"gpu\"\n# est_cnt: int = 6000\n\n# DRY_RUN = True if df_subm.shape[0] == 10 else False\n# if DRY_RUN:\n#     device = \"cpu\"\n#     df_train = df_train.iloc[:50000]\n#     est_cnt: int = 600\n\n# print(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df_train.drop(columns=[\"target\", \"case_id\",\"WEEK_NUM\"])\ny = df_train[\"target\"]\nweeks = df_train[\"WEEK_NUM\"]\n\ndel df_train\ngc.collect()\n\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\n# lgb model parameters\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n#     \"device\": device,\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"max_bin\": 255,\n    \"n_estimators\": 1200,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n#     \"device\": \"gpu\",  # Uncomment if you want to use GPU for training\n}\n\nfitted_models = []\ncv_scores = []\n\n# train and validation set\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    print(\"Valid week range: \", (weeks.iloc[idx_valid].min(), weeks.iloc[idx_valid].max()))\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(50), lgb.early_stopping(50)]\n    )\n\n    fitted_models.append(model)\n\n    y_pred_valid = model.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores.append(auc_score)\n\nmodel = VotingModel(fitted_models)\nprint(\"CV AUC scores: \", cv_scores)\nprint(\"Average CV AUC score: \", sum(cv_scores) / len(cv_scores))\n\ndel X, y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open('lgbm_model.pkl', 'wb') as file:\n     file.write(pickle.dumps(model))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = df_test.drop(columns=[\"WEEK_NUM\"])\nX_test = X_test.set_index(\"case_id\")\n\nlgb_pred = pd.Series(model.predict_proba(X_test)[:, 1], index=X_test.index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## XG Boost","metadata":{}},{"cell_type":"markdown","source":"## Submission\n\nScoring the submission dataset is below, we need to take care of new categories. Then we save the score as a last step. ","metadata":{}},{"cell_type":"code","source":"# df_subm = pd.read_csv(dataPath + \"sample_submission.csv\")\n# df_subm = df_subm.set_index(\"case_id\")\n\n# device: str = \"gpu\"\n# est_cnt: int = 6000\n\n# DRY_RUN = True if df_subm.shape[0] == 10 else False\n# if DRY_RUN:\n#     device = \"cpu\"\n#     df_train = df_train.iloc[:50000]\n#     est_cnt: int = 600\n\n# print(device)\n\ndf_subm[\"score\"] = lgb_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", df_subm[\"score\"].isnull().any())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"submission.csv\")\n\ndel X_test, lgb_pred, df_subm\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Best of luck, and most importantly, enjoy the process of learning and discovery! \n\n<img src=\"https://i.imgur.com/obVWIBh.png\" alt=\"Image\" width=\"700\"/>","metadata":{}}]}