{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":33095,"sourceType":"modelInstanceVersion","modelInstanceId":27710,"modelId":39234},{"sourceId":33096,"sourceType":"modelInstanceVersion","modelInstanceId":27711,"modelId":39234}],"dockerImageVersionId":30683,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport sys\nimport subprocess\nimport os\nimport gc\nfrom pathlib import Path\nfrom glob import glob \nfrom datetime import datetime\nimport seaborn as sns \nimport matplotlib.pyplot as plt\nimport joblib\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.impute import KNNImputer\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \n\nROOT_PATH = Path(\"/kaggle/input/home-credit-credit-risk-model-stability/\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-03T03:10:05.430853Z","iopub.execute_input":"2024-06-03T03:10:05.431653Z","iopub.status.idle":"2024-06-03T03:10:05.438549Z","shell.execute_reply.started":"2024-06-03T03:10:05.431620Z","shell.execute_reply":"2024-06-03T03:10:05.437657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df","metadata":{"execution":{"iopub.status.busy":"2024-06-03T03:10:29.402454Z","iopub.execute_input":"2024-06-03T03:10:29.403077Z","iopub.status.idle":"2024-06-03T03:10:29.413719Z","shell.execute_reply.started":"2024-06-03T03:10:29.403044Z","shell.execute_reply":"2024-06-03T03:10:29.412822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    #Please add or subtract features yourself, be aware that too many features will take up too much space.\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n    \n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return  expr_max\n    \n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return  expr_max\n    \n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return  expr_max \n    \n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] \n        return  expr_max\n    \n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.421719Z","iopub.execute_input":"2024-05-21T13:18:48.422089Z","iopub.status.idle":"2024-05-21T13:18:48.436474Z","shell.execute_reply.started":"2024-05-21T13:18:48.422057Z","shell.execute_reply":"2024-05-21T13:18:48.435548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.439176Z","iopub.execute_input":"2024-05-21T13:18:48.439464Z","iopub.status.idle":"2024-05-21T13:18:48.452219Z","shell.execute_reply.started":"2024-05-21T13:18:48.439441Z","shell.execute_reply":"2024-05-21T13:18:48.451434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(regex_path, depth=None):\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.453355Z","iopub.execute_input":"2024-05-21T13:18:48.45363Z","iopub.status.idle":"2024-05-21T13:18:48.466923Z","shell.execute_reply.started":"2024-05-21T13:18:48.453606Z","shell.execute_reply":"2024-05-21T13:18:48.466079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base.with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.468013Z","iopub.execute_input":"2024-05-21T13:18:48.468287Z","iopub.status.idle":"2024-05-21T13:18:48.477341Z","shell.execute_reply.started":"2024-05-21T13:18:48.468264Z","shell.execute_reply":"2024-05-21T13:18:48.476546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.478659Z","iopub.execute_input":"2024-05-21T13:18:48.479311Z","iopub.status.idle":"2024-05-21T13:18:48.491639Z","shell.execute_reply.started":"2024-05-21T13:18:48.479277Z","shell.execute_reply":"2024-05-21T13:18:48.490796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.492892Z","iopub.execute_input":"2024-05-21T13:18:48.493707Z","iopub.status.idle":"2024-05-21T13:18:48.506524Z","shell.execute_reply.started":"2024-05-21T13:18:48.493673Z","shell.execute_reply":"2024-05-21T13:18:48.505561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = ROOT_PATH / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT_PATH / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.50769Z","iopub.execute_input":"2024-05-21T13:18:48.508057Z","iopub.status.idle":"2024-05-21T13:18:48.520172Z","shell.execute_reply.started":"2024-05-21T13:18:48.508031Z","shell.execute_reply":"2024-05-21T13:18:48.519272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:18:48.524379Z","iopub.execute_input":"2024-05-21T13:18:48.524629Z","iopub.status.idle":"2024-05-21T13:20:01.258496Z","shell.execute_reply.started":"2024-05-21T13:18:48.524608Z","shell.execute_reply":"2024-05-21T13:20:01.257672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\ndel data_store\ngc.collect()\ndf_train = df_train.pipe(Pipeline.filter_cols)\ndf_train, cat_cols = to_pandas(df_train)\ndf_train = reduce_mem_usage(df_train)\nnums=df_train.select_dtypes(exclude='category').columns\nfrom itertools import combinations, permutations\nnans_df = df_train[nums].isna()\nnans_groups={}\nfor col in nums:\n    cur_group = nans_df[col].sum()\n    try:\n        nans_groups[cur_group].append(col)\n    except:\n        nans_groups[cur_group]=[col]\ndel nans_df; x=gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:20:01.259627Z","iopub.execute_input":"2024-05-21T13:20:01.259922Z","iopub.status.idle":"2024-05-21T13:20:48.524838Z","shell.execute_reply.started":"2024-05-21T13:20:01.259898Z","shell.execute_reply":"2024-05-21T13:20:48.524048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_group(grps):\n    use = []\n    for g in grps:\n        mx = 0; vx = g[0]\n        for gg in g:\n            n = df_train[gg].nunique()\n            if n>mx:\n                mx = n\n                vx = gg\n        use.append(vx)\n    return use","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:20:48.526001Z","iopub.execute_input":"2024-05-21T13:20:48.526586Z","iopub.status.idle":"2024-05-21T13:20:48.532228Z","shell.execute_reply.started":"2024-05-21T13:20:48.52655Z","shell.execute_reply":"2024-05-21T13:20:48.531353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def group_columns_by_correlation(matrix, threshold=0.85):\n    correlation_matrix = matrix.corr()\n    groups = []\n    remaining_cols = list(matrix.columns)\n    while remaining_cols:\n        col = remaining_cols.pop(0)\n        group = [col]\n        correlated_cols = [col]\n        for c in remaining_cols:\n            if correlation_matrix.loc[col, c] >= threshold:\n                group.append(c)\n                correlated_cols.append(c)\n        groups.append(group)\n        remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n    \n    return groups","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:20:48.533295Z","iopub.execute_input":"2024-05-21T13:20:48.533571Z","iopub.status.idle":"2024-05-21T13:20:48.545704Z","shell.execute_reply.started":"2024-05-21T13:20:48.533549Z","shell.execute_reply":"2024-05-21T13:20:48.544867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uses=[]\nfor k,v in nans_groups.items():\n    if len(v)>1:\n            Vs = nans_groups[k]\n            grps= group_columns_by_correlation(df_train[Vs], threshold=0.85)\n            use=reduce_group(grps)\n            uses=uses+use\n    else:\n        uses=uses+v\ndf_train = df_train[uses]    ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:20:48.546791Z","iopub.execute_input":"2024-05-21T13:20:48.547053Z","iopub.status.idle":"2024-05-21T13:21:10.376904Z","shell.execute_reply.started":"2024-05-21T13:20:48.54703Z","shell.execute_reply":"2024-05-21T13:21:10.375947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:10.378068Z","iopub.execute_input":"2024-05-21T13:21:10.378387Z","iopub.status.idle":"2024-05-21T13:21:10.616931Z","shell.execute_reply.started":"2024-05-21T13:21:10.378362Z","shell.execute_reply":"2024-05-21T13:21:10.616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\ndel data_store\ngc.collect()\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\ndf_test, cat_cols = to_pandas(df_test)\ndf_test = reduce_mem_usage(df_test)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:10.618221Z","iopub.execute_input":"2024-05-21T13:21:10.618801Z","iopub.status.idle":"2024-05-21T13:21:11.009451Z","shell.execute_reply.started":"2024-05-21T13:21:10.618769Z","shell.execute_reply":"2024-05-21T13:21:11.008443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['target']=0\ndf_test['target']=1","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:11.010503Z","iopub.execute_input":"2024-05-21T13:21:11.010784Z","iopub.status.idle":"2024-05-21T13:21:11.016929Z","shell.execute_reply.started":"2024-05-21T13:21:11.010759Z","shell.execute_reply":"2024-05-21T13:21:11.016195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=pd.concat([df_train,df_test])\ndf_train=reduce_mem_usage(df_train)\n\ny = df_train[\"target\"]\ndf_train= df_train.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])\n\n\njoblib.dump((df_train,y,df_test),'data.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:11.018278Z","iopub.execute_input":"2024-05-21T13:21:11.018625Z","iopub.status.idle":"2024-05-21T13:21:22.775523Z","shell.execute_reply.started":"2024-05-21T13:21:11.018595Z","shell.execute_reply":"2024-05-21T13:21:22.774591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n                if isnull > 0.7:\n                    df = df.drop(col)\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:22.77686Z","iopub.execute_input":"2024-05-21T13:21:22.777248Z","iopub.status.idle":"2024-05-21T13:21:22.790333Z","shell.execute_reply.started":"2024-05-21T13:21:22.777215Z","shell.execute_reply":"2024-05-21T13:21:22.789276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n        expr_var = [pl.var(col).alias(f\"var_{col}\") for col in cols]\n\n        return expr_max + expr_last + expr_mean \n\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        expr_median = [pl.median(col).alias(f\"median_{col}\") for col in cols]\n\n        return expr_max + expr_last + expr_mean \n\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        # expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n        return expr_max + expr_last  # +expr_count\n\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return expr_max + expr_last\n\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        # expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        # expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return expr_max + expr_last\n\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:22.791597Z","iopub.execute_input":"2024-05-21T13:21:22.791889Z","iopub.status.idle":"2024-05-21T13:21:22.808629Z","shell.execute_reply.started":"2024-05-21T13:21:22.791864Z","shell.execute_reply":"2024-05-21T13:21:22.807748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df\n\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:22.809683Z","iopub.execute_input":"2024-05-21T13:21:22.809998Z","iopub.status.idle":"2024-05-21T13:21:22.825719Z","shell.execute_reply.started":"2024-05-21T13:21:22.809973Z","shell.execute_reply":"2024-05-21T13:21:22.824934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:22.826732Z","iopub.execute_input":"2024-05-21T13:21:22.826984Z","iopub.status.idle":"2024-05-21T13:21:22.839982Z","shell.execute_reply.started":"2024-05-21T13:21:22.826963Z","shell.execute_reply":"2024-05-21T13:21:22.83908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_notebook_info = joblib.load('/kaggle/input/homecredit-models-public/other/lgb/1/notebook_info.joblib')\nprint(f\"- [lgb] notebook_start_time: {lgb_notebook_info['notebook_start_time']}\")\nprint(f\"- [lgb] description: {lgb_notebook_info['description']}\")\n\ncols = lgb_notebook_info['cols']\ncat_cols = lgb_notebook_info['cat_cols']\nprint(f\"- [lgb] len(cols): {len(cols)}\")\nprint(f\"- [lgb] len(cat_cols): {len(cat_cols)}\")\nlgb_models = joblib.load('/kaggle/input/homecredit-models-public/other/lgb/1/lgb_models.joblib')\nlgb_models\n\ncat_notebook_info = joblib.load('/kaggle/input/homecredit-models-public/other/cat/1/notebook_info.joblib')\nprint(f\"- [cat] notebook_start_time: {cat_notebook_info['notebook_start_time']}\")\nprint(f\"- [cat] description: {cat_notebook_info['description']}\")\n\ncat_models = joblib.load('/kaggle/input/homecredit-models-public/other/cat/1/cat_models.joblib')\ncat_models","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:22.841016Z","iopub.execute_input":"2024-05-21T13:21:22.841306Z","iopub.status.idle":"2024-05-21T13:21:28.468502Z","shell.execute_reply.started":"2024-05-21T13:21:22.841284Z","shell.execute_reply":"2024-05-21T13:21:28.467466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DIR = ROOT_PATH / \"parquet_files\" / \"test\"\n\ndata_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TEST_DIR / \"test_applprev_2.parquet\", 2),\n        read_file(TEST_DIR / \"test_person_2.parquet\", 2)\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:28.46972Z","iopub.execute_input":"2024-05-21T13:21:28.470106Z","iopub.status.idle":"2024-05-21T13:21:28.679719Z","shell.execute_reply.started":"2024-05-21T13:21:28.470069Z","shell.execute_reply":"2024-05-21T13:21:28.678806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\nprint(\"test data shape:\\t\", df_test.shape)\ndel data_store\ngc.collect()\n\n\ndf_test = df_test.select(['case_id'] + cols)\n\ndf_test, cat_cols = to_pandas(df_test, cat_cols)\ndf_test = reduce_mem_usage(df_test)\ndf_test = df_test.set_index('case_id')\nprint(\"test data shape:\\t\", df_test.shape)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:28.680847Z","iopub.execute_input":"2024-05-21T13:21:28.681502Z","iopub.status.idle":"2024-05-21T13:21:29.205122Z","shell.execute_reply.started":"2024-05-21T13:21:28.681446Z","shell.execute_reply":"2024-05-21T13:21:29.204138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n\n    def __init__(self, estimators):\n\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n     \n    def predict_proba(self, X):      \n\n        # lgb\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]\n        \n        # cat\n        X[cat_cols] = X[cat_cols].astype(str)\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[-5:]]\n        \n        return np.mean(y_preds, axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:29.206561Z","iopub.execute_input":"2024-05-21T13:21:29.206921Z","iopub.status.idle":"2024-05-21T13:21:29.215116Z","shell.execute_reply.started":"2024-05-21T13:21:29.206889Z","shell.execute_reply":"2024-05-21T13:21:29.214121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = VotingModel(lgb_models + cat_models)\nlen(model.estimators)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:29.216688Z","iopub.execute_input":"2024-05-21T13:21:29.217058Z","iopub.status.idle":"2024-05-21T13:21:29.231076Z","shell.execute_reply.started":"2024-05-21T13:21:29.217028Z","shell.execute_reply":"2024-05-21T13:21:29.230202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = pd.Series(model.predict_proba(df_test)[:, 1], index=df_test.index)\ndf_subm = pd.read_csv(ROOT_PATH / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\ndf_subm[\"score\"] = y_pred\ndf_subm.to_csv(\"sub.csv\")\ndf_train,y,df_test=joblib.load('/kaggle/working/data.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:29.236293Z","iopub.execute_input":"2024-05-21T13:21:29.236565Z","iopub.status.idle":"2024-05-21T13:21:30.34102Z","shell.execute_reply.started":"2024-05-21T13:21:29.236541Z","shell.execute_reply":"2024-05-21T13:21:30.340191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fitted_models_lgb=[]\nmodel = lgb.LGBMClassifier()\nmodel.fit(df_train,y)\nfitted_models_lgb.append(model) ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:21:30.342184Z","iopub.execute_input":"2024-05-21T13:21:30.342486Z","iopub.status.idle":"2024-05-21T13:23:13.510537Z","shell.execute_reply.started":"2024-05-21T13:21:30.342461Z","shell.execute_reply":"2024-05-21T13:23:13.50955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n    def fit(self, X, y=None):\n        return self\n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\nmodel = VotingModel(fitted_models_lgb)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:23:13.512005Z","iopub.execute_input":"2024-05-21T13:23:13.512469Z","iopub.status.idle":"2024-05-21T13:23:13.51958Z","shell.execute_reply.started":"2024-05-21T13:23:13.512435Z","shell.execute_reply":"2024-05-21T13:23:13.518657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.drop(columns=[\"WEEK_NUM\",'target'])\ndf_test = df_test.set_index(\"case_id\")\n\ny_pred = pd.Series(model.predict_proba(df_test)[:,1], index=df_test.index)\ncondition=y_pred<0.98\ndf_subm = pd.read_csv(\"/kaggle/working/sub.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm.loc[condition, 'score'] = (df_subm.loc[condition, 'score'] - 0.073).clip(0)\ndf_subm.to_csv(\"submission.csv\")\n!rm -rf data.pkl","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:23:13.520711Z","iopub.execute_input":"2024-05-21T13:23:13.521482Z","iopub.status.idle":"2024-05-21T13:23:14.822507Z","shell.execute_reply.started":"2024-05-21T13:23:13.52145Z","shell.execute_reply":"2024-05-21T13:23:14.820933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}