{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture captured_output\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:17.707691Z","iopub.execute_input":"2026-06-18T13:31:17.707969Z","iopub.status.idle":"2026-06-18T13:31:18.554861Z","shell.execute_reply.started":"2026-06-18T13:31:17.707940Z","shell.execute_reply":"2026-06-18T13:31:18.553774Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"一、数据预处理","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.metrics import roc_auc_score\nfrom pathlib import Path\nimport gc   \nimport warnings\nimport joblib\nwarnings.filterwarnings('ignore')\n\nDATA_PATH = Path(\"/kaggle/input/competitions/home-credit-credit-risk-model-stability/parquet_files/\")\nOUTPUT_PATH = Path(\"/kaggle/working/processed\")\nOUTPUT_PATH.mkdir(exist_ok=True)\nCACHE_DIR = Path(\"feature_cache\")\nCACHE_DIR.mkdir(exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:18.556677Z","iopub.execute_input":"2026-06-18T13:31:18.557256Z","iopub.status.idle":"2026-06-18T13:31:24.537858Z","shell.execute_reply.started":"2026-06-18T13:31:18.557212Z","shell.execute_reply":"2026-06-18T13:31:24.537020Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**1. 数据选取**","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n    \"train\": {\n        \"base\": \"train_base.parquet\",\n\n        \"static_0\": [\n            \"train_static_0_0.parquet\",\n            \"train_static_0_1.parquet\"\n        ],\n\n        \"static_cb\": [\n            \"train_static_cb_0.parquet\"\n        ],\n\n        \"applprev\": [\n            \"train_applprev_1_0.parquet\",\n            \"train_applprev_1_1.parquet\"\n        ],\n        \"cba1\": [\n            \"train_credit_bureau_a_1_0.parquet\",\n            \"train_credit_bureau_a_1_1.parquet\",\n            \"train_credit_bureau_a_1_2.parquet\",\n            \"train_credit_bureau_a_1_3.parquet\"\n        ],\n        \"cbb1\": [\n            \"train_credit_bureau_b_1.parquet\"\n        ],\n        \"deposit\": [\n            \"train_deposit_1.parquet\"\n        ],\n        \"person\": [\n            \"train_person_1.parquet\"\n        ],\n        \"debit\": [\n            \"train_debitcard_1.parquet\"\n        ],\n        \"applprev2\": \"train_applprev_2.parquet\"\n    },\n    \n    \"test\": {\n        \"base\": \"test_base.parquet\",\n\n        \"static_0\": [\n            \"test_static_0_0.parquet\",\n            \"test_static_0_1.parquet\",\n            \"test_static_0_2.parquet\"\n        ],\n\n        \"static_cb\": [\n            \"test_static_cb_0.parquet\"\n        ],\n        \n        \"applprev\": [\n            \"test_applprev_1_0.parquet\",\n            \"test_applprev_1_1.parquet\",\n            \"test_applprev_1_2.parquet\"\n        ],\n        \"cba1\": [\n            \"test_credit_bureau_a_1_0.parquet\",\n            \"test_credit_bureau_a_1_1.parquet\",\n            \"test_credit_bureau_a_1_2.parquet\",\n            \"test_credit_bureau_a_1_3.parquet\",\n            \"test_credit_bureau_a_1_4.parquet\"\n        ],\n        \"cbb1\": [\n            \"test_credit_bureau_b_1.parquet\"\n        ],\n        \"deposit\": [\n            \"test_deposit_1.parquet\"\n        ],\n        \"person\": [\n            \"test_person_1.parquet\"\n        ],\n        \"debit\": [\n            \"test_debitcard_1.parquet\"\n        ],\n        \"applprev2\": \"test_applprev_2.parquet\"\n    }\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:24.538912Z","iopub.execute_input":"2026-06-18T13:31:24.539235Z","iopub.status.idle":"2026-06-18T13:31:24.546058Z","shell.execute_reply.started":"2026-06-18T13:31:24.539208Z","shell.execute_reply":"2026-06-18T13:31:24.545206Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**2. 数据聚合**","metadata":{}},{"cell_type":"code","source":"def set_table_dtypes(df):\n    if 'case_id' in df.columns:\n        df = df.with_columns(pl.col('case_id').cast(pl.Int32, strict=False))\n    return df\n\ndef load_and_merge_table(train_files,table_name,mode):\n    \"\"\"\n    加载并合并训练集多个 parquet 分片\n    \"\"\"\n    print(f\"加载 {table_name} ...\")\n    try:\n        train_df = None\n        for f in train_files:\n            full_path = DATA_PATH / mode / f\n            if not full_path.exists():\n                print(f\"    文件不存在: {full_path}\")\n                continue\n            df = pl.read_parquet(full_path)\n            if train_df is None:\n                train_df = df\n            else:\n                train_df = pl.concat([train_df, df],how=\"vertical\")\n        if train_df is None:\n            print(\"    没有成功加载任何文件\")\n            return None\n        # 仅统一 case_id 类型\n        if \"case_id\" in train_df.columns:\n            train_df = train_df.with_columns(\n                pl.col(\"case_id\").cast(pl.Int32, strict=False).alias(\"case_id\"))\n        print(f\"    合并后形状: {train_df.shape}\")\n        return train_df\n    except Exception as e:\n        print(f\"    加载失败: {e}\")\n        return None\n        \ndef apply_time_filter(df: pl.DataFrame, date_col: str, case_dates: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"时间过滤：只保留 date_col < 申请日期的记录\"\"\"\n    if date_col not in df.columns:\n        return df\n    # 确保日期列是字符串格式\n    if df[date_col].dtype != pl.Utf8:\n        df = df.with_columns(pl.col(date_col).cast(pl.Utf8).alias(date_col))\n    # 确保 case_dates 中的 date_decision 也是字符串\n    case_dates_copy = case_dates.select([\n        'case_id',\n        pl.col('date_decision').cast(pl.Utf8).alias('date_decision')\n    ])\n    df = df.join(case_dates_copy, on='case_id', how='left')\n    df = df.filter(pl.col(date_col) < pl.col('date_decision'))\n    df = df.drop('date_decision')\n\n    return df\n\ndef aggregate_depth1(\n    df: pl.DataFrame,\n    group_col: str = \"case_id\",\n    prefix: str = \"\"\n):\n    agg_exprs = []\n    exclude_cols = {\n        \"case_id\",\n        \"num_group1\",\n        \"num_group2\"\n    }\n    for col in df.columns:\n        if col in exclude_cols:\n            continue\n        suffix = col[-1]\n        dtype = df.schema[col]\n        if suffix in (\"A\", \"P\"):\n            agg_exprs.extend([\n                #pl.col(col).count().alias(f\"{prefix}{col}_count\"),\n                #pl.col(col).sum().alias(f\"{prefix}{col}_sum\"),\n                pl.col(col).mean().alias(f\"{prefix}{col}_mean\"),\n                pl.col(col).max().alias(f\"{prefix}{col}_max\"),\n                pl.col(col).min().alias(f\"{prefix}{col}_min\"),\n                #pl.col(col).std().alias(f\"{prefix}{col}_std\"),\n            ])\n        elif suffix == \"L\":\n            if dtype.is_numeric():\n                agg_exprs.extend([\n                    #pl.col(col).count().alias(f\"{prefix}{col}_count\"),\n                    #pl.col(col).sum().alias(f\"{prefix}{col}_sum\"),\n                    pl.col(col).mean().alias(f\"{prefix}{col}_mean\"),\n                    pl.col(col).max().alias(f\"{prefix}{col}_max\"),\n                    pl.col(col).min().alias(f\"{prefix}{col}_min\"),\n                    #pl.col(col).std().alias(f\"{prefix}{col}_std\"),\n                ])\n            else:\n                agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        elif suffix == \"M\":\n            agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        elif suffix == \"D\":\n            agg_exprs.extend([\n                pl.col(col).max().alias(f\"{prefix}{col}_max\"),\n                pl.col(col).min().alias(f\"{prefix}{col}_min\"),\n            ])\n    result = (df.group_by(group_col).agg(agg_exprs))\n\n    return result\n\ndef process_depth1_table(file_path,table_name,prefix=\"\"):\n    print(f\"处理 {table_name}\")\n    df = pl.read_parquet(file_path)\n    if \"case_id\" in df.columns:\n        df = df.with_columns(pl.col(\"case_id\").cast(pl.Int32, strict=False))\n    agg_df = aggregate_depth1(df,group_col=\"case_id\",prefix=prefix)\n    print(f\"    原始: {df.shape} -> 聚合后: {agg_df.shape}\")\n    del df\n    gc.collect()\n\n    return agg_df\n    \ndef process_depth1_files(mode,file_list,feature_name,prefix=\"\"):\n    agg_parts = []\n    for file_name in file_list:\n        path = DATA_PATH / mode / file_name\n        if not path.exists():\n            continue\n        agg_df = process_depth1_table(path,file_name,prefix)\n        agg_parts.append(agg_df)\n    if len(agg_parts) == 0:\n        return None\n    result = pl.concat(agg_parts,how=\"vertical_relaxed\")\n    print(f\"    {feature_name} 最终形状: {result.shape}\")\n    save_feature_table(result,feature_name)\n    del agg_parts\n    gc.collect()\n\n    return result\n\ndef save_feature_table(df,feature_name):\n    path = CACHE_DIR / f\"{feature_name}.parquet\"\n    df.write_parquet(path)\n    print(f\"    保存成功: {path}\")\n    \n    return path\n\ndef load_feature_table(feature_name):\n    path = CACHE_DIR / f\"{feature_name}.parquet\"\n    \n    return pl.read_parquet(path)\n\ndef aggregate_depth2_level1(df,prefix: \"applprev2\"):\n    agg_exprs = []\n    exclude_cols = {\n        \"case_id\",\n        \"num_group1\",\n        \"num_group2\"\n    }\n    for col in df.columns:\n        if col in exclude_cols:\n            continue\n        suffix = col[-1]\n        dtype = df.schema[col]\n        # A/P\n        if suffix in (\"A\", \"P\"):\n            agg_exprs.extend([\n                #pl.col(col).count().alias(f\"{col}_count\"),\n                #pl.col(col).sum().alias(f\"{col}_sum\"),\n                pl.col(col).mean().alias(f\"{col}_mean\"),\n                pl.col(col).max().alias(f\"{col}_max\"),\n                pl.col(col).min().alias(f\"{col}_min\"),\n                #pl.col(col).std().alias(f\"{col}_std\"),\n            ])\n        # L 自动识别\n        elif suffix == \"L\":\n            if dtype.is_numeric():\n                agg_exprs.extend([\n                    #pl.col(col).count().alias(f\"{col}_count\"),\n                    #pl.col(col).sum().alias(f\"{col}_sum\"),\n                    pl.col(col).mean().alias(f\"{col}_mean\"),\n                    pl.col(col).max().alias(f\"{col}_max\"),\n                    pl.col(col).min().alias(f\"{col}_min\"),\n                    #pl.col(col).std().alias(f\"{col}_std\"),\n                ])\n            else:\n                agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        # M\n        elif suffix == \"M\":\n            agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        # D\n        elif suffix == \"D\":\n            agg_exprs.extend([\n                pl.col(col).max().alias(f\"{col}_max\"),\n                pl.col(col).min().alias(f\"{col}_min\"),\n            ])\n            \n    return (df.group_by([\"case_id\", \"num_group1\"]).agg(agg_exprs))\n    \ndef aggregate_depth2_level2(df):\n    agg_cols = [c for c in df.columns if c not in (\"case_id\", \"num_group1\")]\n    agg_exprs = []\n    for col in agg_cols:\n        agg_exprs.extend([\n            pl.col(col).mean().alias(f\"{col}_mean_across_contracts\"),\n            pl.col(col).max().alias(f\"{col}_max_across_contracts\"),\n            pl.col(col).sum().alias(f\"{col}_sum_across_contracts\"),\n            pl.col(col).std().alias(f\"{col}_std_across_contracts\"),\n        ])\n        \n    return (df.group_by(\"case_id\").agg(agg_exprs))\n\ndef process_applprev2(path,case_dates):\n    print(\"处理 applprev_2\")\n    df = pl.read_parquet(path)\n    print(\"原始:\", df.shape)\n    # 时间过滤\n    if \"creationdate_885D\" in df.columns:\n        df = apply_time_filter(df,\"creationdate_885D\",case_dates)\n        print(\"时间过滤后:\", df.shape)\n    level1 = aggregate_depth2_level1(df,prefix=\"applprev2\")\n    print(\"第一层聚合:\",level1.shape)\n    save_feature_table(level1,\"applprev2_level1\")\n    del df\n    del level1\n    gc.collect()\n    level1 = load_feature_table(\"applprev2_level1\")\n    level2 = aggregate_depth2_level2(level1)\n    print(\"第二层聚合:\",level2.shape)\n    save_feature_table(level2,\"applprev2_final\")\n    del level1\n    gc.collect()\n\n    return None\n\n#数据聚合\ndef build_dataset(mode=\"train\"):\n    ##base表\n    cfg = CONFIG[mode]\n    base_file = cfg[\"base\"]\n    base_file = pl.read_parquet(DATA_PATH / mode / base_file)\n    print(f\"base表形状: {base_file.shape}\")\n    cols = [\n        pl.col(\"case_id\").cast(pl.Int32, strict=False),\n        pl.col(\"WEEK_NUM\").cast(pl.Int32, strict=False),\n        pl.col(\"date_decision\").cast(pl.Date, strict=False),\n    ]\n    if \"target\" in base_file.columns:\n        cols.append(pl.col(\"target\").cast(pl.Int8, strict=False))\n    base = base_file.with_columns(cols)\n    key_cols = [\n        \"case_id\",\n        \"target\",\n        \"date_decision\",\n        \"WEEK_NUM\"\n    ]\n    base = base.select([c for c in key_cols if c in base.columns])\n    base = base.select([c for c in key_cols if c in base.columns])\n    print(f\"base表形状: {base.shape}\")\n    case_dates = base.select([\"case_id\", \"date_decision\"])\n    del base_file\n    gc.collect()\n    \n    ##depth=0\n    static_0 = load_and_merge_table(cfg[\"static_0\"],\"static_0\",mode)\n    base = base.join(static_0,on=\"case_id\",how=\"left\")\n    print(f\"    新增 {static_0.shape[1] - 1} 列\")\n    del static_0\n    gc.collect()\n    static_cb = load_and_merge_table(cfg[\"static_cb\"],\"static_cb_0\",mode)\n    base = base.join(static_cb,on=\"case_id\",how=\"left\")\n    print(f\"    新增 {static_cb.shape[1] - 1} 列\")\n    del static_cb\n    gc.collect()\n    print(f\"    当前base表列数: {base.shape[1]}\")\n    print(f\"    当前base表行数: {base.shape[0]}\")\n    \n    ##depth=1\n    DEPTH1_TABLES = {k: cfg[k] for k in [\"applprev\",\"cba1\",\"cbb1\",\"deposit\",\"person\",\"debit\"]}\n    for prefix, files in DEPTH1_TABLES.items():\n        print(f\"处理 {prefix}\")\n        agg_df = process_depth1_files(mode,files,feature_name=prefix,prefix=f\"{prefix}_\")\n        if agg_df is None:\n            continue\n        # 保存聚合结果\n        agg_path = CACHE_DIR / f\"{prefix}_agg.parquet\"\n        agg_df.write_parquet(agg_path)\n        print(f\"    聚合结果已保存: {agg_path}\")\n        print(f\"    聚合结果形状: {agg_df.shape}\")\n        before_cols = base.shape[1]\n        #base\n        base = base.join(agg_df,on=\"case_id\",how=\"left\")\n        after_cols = base.shape[1]\n        print(f\"    新增 {after_cols - before_cols} 列\")\n        print(f\"    当前 base: {base.shape}\")\n        del agg_df\n        gc.collect()\n        base_path = CACHE_DIR / f\"base_after_{prefix}.parquet\"\n        base.write_parquet(base_path)\n        print(f\"    当前base已保存: {base_path}\")\n        del base\n        gc.collect()\n        base = pl.read_parquet(base_path)\n        print(f\"重新加载base: {base.shape}\")\n    \n    ##depth=2\n    path = DATA_PATH / mode / cfg[\"applprev2\"]\n    process_applprev2(path,case_dates)\n    applprev2 = load_feature_table(\"applprev2_final\")\n    rename_map = {c: f\"applprev2_{c}\" for c in applprev2.columns if c != \"case_id\"}\n    applprev2 = applprev2.rename(rename_map)\n    #base\n    before_cols = base.shape[1]\n    base = base.join(applprev2,on=\"case_id\",how=\"left\")\n    after_cols = base.shape[1]\n    print(f\"    新增 {after_cols-before_cols} 列\")\n    save_feature_table(base,\"base_after_applprev2\")\n    del applprev2\n    gc.collect()\n\n    return base","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:24.548252Z","iopub.execute_input":"2026-06-18T13:31:24.548490Z","iopub.status.idle":"2026-06-18T13:31:24.592699Z","shell.execute_reply.started":"2026-06-18T13:31:24.548469Z","shell.execute_reply":"2026-06-18T13:31:24.591871Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**3. 数据处理与特征工程**","metadata":{}},{"cell_type":"code","source":"def clean_missing_and_constant_features(\n    df,\n    missing_threshold=0.90,\n    save_report=True\n):\n    \"\"\"\n    删除：\n    1. 全空列\n    2. 高缺失列（默认>90%）\n    3. 单值列\n    4. 单值+空值列\n\n    返回：\n    cleaned_df\n    report_df\n    \"\"\"\n    n_rows = len(df)\n    drop_cols = []\n    report = []\n    for col in df.columns:\n        if col in [\"case_id\", \"target\"]:\n            continue\n        s = df[col]\n        null_ratio = s.null_count() / n_rows\n        try:\n            nunique = s.drop_nulls().n_unique()\n        except:\n            nunique = None\n        reason = None\n        if null_ratio == 1:\n            reason = \"all_null\"\n        elif null_ratio > missing_threshold:\n            reason = \"high_missing\"\n        elif nunique == 1 and null_ratio == 0:\n            reason = \"constant\"\n        elif nunique <= 1:\n            reason = \"constant_plus_null\"\n        if reason is not None:\n            drop_cols.append(col)\n            report.append({\n                \"column\": col,\n                \"reason\": reason,\n                \"null_ratio\": round(null_ratio, 6),\n                \"nunique\": nunique,\n                \"dtype\": str(df.schema[col])\n            })    \n    print(f\"低质数据处理-待删除列数: {len(drop_cols)}\")\n    cleaned_df = df.drop(drop_cols)\n    print(f\"    处理前: {df.shape}\")\n    print(f\"    处理后: {cleaned_df.shape}\")\n    report_df = pl.DataFrame(report)\n    if save_report:\n        report_df.write_csv(\"drop_feature_report.csv\")\n        print(\"    删除报告已保存: drop_feature_report.csv\")\n\n    return cleaned_df, report_df, drop_cols\n\ndef convert_dates_to_days(df, decision_col=\"date_decision\"):\n    date_cols = [\n        c\n        for c in df.columns\n        if c.endswith(\"D\")\n        and c != decision_col\n    ]\n    print(f\"日期数据处理-发现 {len(date_cols)} 个日期列\")\n    for col in date_cols:\n        print(f\"    处理 {col}\")\n        dtype = df.schema[col]\n        # 已经是 Date\n        if dtype == pl.Date:\n            date_expr = pl.col(col)\n        else:\n            date_expr = (\n                pl.col(col)\n                .cast(pl.String, strict=False)\n                .str.to_date(strict=False)\n            )\n        df = df.with_columns([\n            pl.col(col).is_null().cast(pl.Int8).alias(f\"{col}_missing\"),\n            (date_expr- pl.col(decision_col)).dt.total_days().alias(f\"{col}_days\")\n        ])\n    drop_cols = [c for c in date_cols + [decision_col] if c in df.columns]\n    df = df.drop(drop_cols)\n    print(f\"    删除 {len(drop_cols)} 个原始日期列\")\n\n    return df\n\ndef export_category_summary(df):\n    rows = []\n    cat_cols = []\n    for col in df.columns:\n        dtype = df.schema[col]\n        if dtype == pl.String:\n            cat_cols.append(col)\n            rows.append({\n                \"column\": col,\n                \"nunique\": (df[col].drop_nulls().n_unique()),\n                \"missing_ratio\": (df[col].null_count() / len(df))\n            })\n    summary = (pl.DataFrame(rows).sort(\"nunique\"))\n    summary.write_csv(\"category_summary.csv\")\n    print(f\"类别数据处理-查找到类别列数: {len(cat_cols)}\")\n\n    return summary\n\ndef one_hot_low_cardinality(\n    df,\n    threshold=15,\n    columns=None\n):\n    if columns is None:\n        cat_cols = []\n        for col in df.columns:\n            if df.schema[col] != pl.String:\n                continue\n            nunique = df[col].drop_nulls().n_unique()\n            if nunique <= threshold:\n                cat_cols.append(col)\n    else:\n        cat_cols = [c for c in columns if c in df.columns]\n    print(f\"OHE列数: {len(cat_cols)}\")\n    if len(cat_cols) == 0:\n        return df, cat_cols\n    df = df.to_dummies(columns=cat_cols)\n    return df, cat_cols\n   \ndef drop_high_cardinality_ids(\n    df,\n    unique_threshold=50,\n    columns=None\n):\n    if columns is None:\n        drop_cols = []\n        for col in df.columns:\n            if df.schema[col] != pl.String:\n                continue\n            nunique = df[col].drop_nulls().n_unique()\n            if nunique > unique_threshold:\n                drop_cols.append(col)\n    else:\n        drop_cols = [c for c in columns if c in df.columns]\n    print(f\"删除高基数列: {len(drop_cols)}\")\n    return df.drop(drop_cols), drop_cols\n\ndef create_missing_flags(\n    df,\n    threshold=0.3,\n    columns=None\n):\n    if columns is None:\n        prefixes = (\n            \"cba1_\",\n            \"cbb1_\",\n            \"deposit_\",\n            \"debit_\",\n            \"person_\",\n            \"applprev_\",\n            \"applprev2_\"\n        )\n        suffixes = (\n            \"_mean\",\n            \"_sum\",\n            \"_max\",\n            \"_min\",\n            \"_std\"\n        )\n        flag_cols = []\n        for col in df.columns:\n            if not col.startswith(prefixes):\n                continue\n            if not any(s in col for s in suffixes):\n                continue\n            miss_ratio = (df[col].null_count() / len(df))\n            if miss_ratio > threshold:\n                flag_cols.append(col)\n    else:\n        flag_cols = [c for c in columns if c in df.columns]\n    new_cols = []\n    for col in flag_cols:\n        new_cols.append(pl.col(col).is_null().cast(pl.Int8).alias(f\"{col}_missing\"))\n    if len(new_cols):\n        df = df.with_columns(new_cols)\n    print(f\"新增 Missing Flag: {len(new_cols)}\")\n\n    return df, flag_cols\n\ndef compress_dtypes(df):\n    for col in df.columns:\n        dtype = df.schema[col]\n        if dtype == pl.Float64:\n            df = df.with_columns(pl.col(col).cast(pl.Float32))\n        elif dtype == pl.Int64:\n            df = df.with_columns(pl.col(col).cast(pl.Int32))\n\n    return df\n\ndef corr_filter_by_group(df,profile,corr_threshold=0.98):\n    remove_cols = set()\n    numeric_types = (\n        pl.Int8,\n        pl.Int16,\n        pl.Int32,\n        pl.Int64,\n        pl.Float32,\n        pl.Float64\n    )\n    numeric_cols = [\n        c\n        for c in df.columns\n        if df.schema[c] in numeric_types\n        and c not in [\"case_id\", \"target\"]\n    ]\n    groups = (profile[\"missing_group\"].unique().sort().to_list())\n    \n    for group in groups:\n        print(f\"group {group}\")\n        cols = (profile.filter(pl.col(\"missing_group\") == group)[\"column\"].to_list())\n        cols = [c for c in cols if c in numeric_cols]\n        if len(cols) < 2:\n            continue\n        print(f\"    group内特征数: {len(cols)}\")\n        # 只取当前group\n        tmp = (df.select(cols).to_pandas())\n        corr = tmp.corr().abs()\n        upper = corr.where(np.triu(np.ones(corr.shape),k=1).astype(bool))\n        for col in upper.columns:\n            high_corr = upper.index[upper[col] > corr_threshold]\n            for other in high_corr:\n                nunique_col = (profile.filter(pl.col(\"column\") == col)[\"nunique\"][0])\n                nunique_other = (profile.filter(pl.col(\"column\") == other)[\"nunique\"][0])\n                if nunique_col >= nunique_other:\n                    remove_cols.add(other)\n                else:\n                    remove_cols.add(col)\n        print(f\"    累计删除: {len(remove_cols)}\")\n        del tmp\n        del corr\n        del upper\n        gc.collect()\n        \n    return list(remove_cols)\n\ndef build_feature_profile(df):\n    rows = []\n    n = len(df)\n    for col in df.columns:\n        if col in [\"case_id\", \"target\"]:\n            continue\n        rows.append({\n            \"column\": col,\n            \"missing_ratio\": df[col].null_count() / n,\n            \"nunique\": (df[col].drop_nulls().n_unique()),\n            \"dtype\": str(df.schema[col])\n        })\n    profile = pl.DataFrame(rows)\n    profile.write_csv(\"feature_profile.csv\")\n    print(profile.shape)\n\n    return profile\n\ndef add_missing_group(profile):\n    profile = profile.with_columns((\n            (pl.col(\"missing_ratio\") * 20)\n            .floor()\n            .cast(pl.Int32)\n        ).alias(\"missing_group\")\n    )\n\n    return profile\n\ndef align_test_columns(test_df,train_columns):\n    for col in train_columns:\n        if col not in test_df.columns:\n            test_df = test_df.with_columns(pl.lit(None).alias(col))\n\n    return test_df.select(train_columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:24.593998Z","iopub.execute_input":"2026-06-18T13:31:24.594273Z","iopub.status.idle":"2026-06-18T13:31:24.627382Z","shell.execute_reply.started":"2026-06-18T13:31:24.594248Z","shell.execute_reply":"2026-06-18T13:31:24.626474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#fit for train\ndef fit_preprocess(\n    base,\n    output_path=None,\n    meta_path=\"/kaggle/working/preprocess_meta.pkl\"\n):\n    metadata = {}\n    print(\"===== Step1: Clean =====\")\n    base, drop_report, drop_cols = (\n        clean_missing_and_constant_features(\n            base,\n            missing_threshold=0.90\n        )\n    )\n    metadata[\"drop_cols\"] = drop_cols\n    \n    print(\"===== Step2: Dates =====\")\n    date_cols = [\n        c\n        for c in base.columns\n        if c.endswith(\"D\")\n        and c != \"date_decision\"\n    ]\n    metadata[\"date_cols\"] = date_cols\n    base = convert_dates_to_days(base)\n    \n    print(\"===== Step3: OHE =====\")\n    base, ohe_cols = one_hot_low_cardinality(base,threshold=15)\n    metadata[\"ohe_cols\"] = ohe_cols\n    \n    print(\"===== Step4: High Cardinality =====\")\n    base, high_card_cols = (drop_high_cardinality_ids(base,unique_threshold=50))\n    metadata[\"high_card_cols\"] = high_card_cols\n    \n    print(\"===== Step5: Missing Flags =====\")\n    base, missing_flag_cols = (create_missing_flags(base,threshold=0.30))\n    metadata[\"missing_flag_cols\"] = missing_flag_cols\n    \n    print(\"===== Step6: Compress =====\")\n    base = compress_dtypes(base)\n    \n    print(\"===== Step7: Correlation =====\")\n    feature_profile = build_feature_profile(base)\n    feature_profile = add_missing_group(feature_profile)\n    corr_sample = base.sample(n=min(1000000, len(base)),seed=42)\n    drop_corr_cols = corr_filter_by_group(\n        corr_sample,\n        feature_profile,\n        corr_threshold=0.98\n    )\n    metadata[\"drop_corr_cols\"] = drop_corr_cols\n    base = base.drop([c for c in drop_corr_cols if c in base.columns])\n    metadata[\"final_columns\"] = base.columns\n    print(f\"最终训练集形状: {base.shape}\")\n    joblib.dump(\n        metadata,\n        meta_path\n    )\n    print(f\"metadata已保存: {meta_path}\")\n    if output_path:\n        base.write_parquet(output_path)\n\n    return base\n\n#transform for test\ndef transform_preprocess(\n    base,\n    meta_path=\"/kaggle/working/preprocess_meta.pkl\",\n    output_path=None\n):\n    metadata = joblib.load(meta_path)\n    print(\"===== Step1: Drop Training Removed Columns =====\")\n    base = base.drop([c for c in metadata[\"drop_cols\"] if c in base.columns])\n\n    print(\"===== Step2: Dates =====\")\n    base = convert_dates_to_days(base)\n\n    print(\"===== Step3: OHE =====\")\n    base, _ = one_hot_low_cardinality(\n        base,\n        columns=metadata[\"ohe_cols\"]\n    )\n\n    print(\"===== Step4: High Cardinality =====\")\n    base, _ = drop_high_cardinality_ids(\n        base,\n        columns=metadata[\"high_card_cols\"]\n    )\n\n    print(\"===== Step5: Missing Flags =====\")\n    base, _ = create_missing_flags(\n        base,\n        columns=metadata[\"missing_flag_cols\"]\n    )\n    print(\"===== Step6: Compress =====\")\n    base = compress_dtypes(base)\n    \n    print(\"===== Step7: Correlation Columns =====\")\n    base = base.drop([c for c in metadata[\"drop_corr_cols\"] if c in base.columns])\n    \n    print(\"===== Step8: Align Columns =====\")\n    train_cols = metadata[\"final_columns\"]\n    for col in train_cols:\n        if col not in base.columns:\n            base = base.with_columns(\n                pl.lit(None).alias(col)\n            )\n    base = base.select(train_cols)\n    base = base.drop('target')\n    print(f\"最终测试集形状: {base.shape}\")\n    if output_path:\n        base.write_parquet(output_path)\n\n    return base","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:24.628477Z","iopub.execute_input":"2026-06-18T13:31:24.628748Z","iopub.status.idle":"2026-06-18T13:31:24.646837Z","shell.execute_reply.started":"2026-06-18T13:31:24.628726Z","shell.execute_reply":"2026-06-18T13:31:24.645945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"===================训练集聚合===================\")\ntrain_base = build_dataset(\"train\")\nprint(\"\\n=================测试集聚合===================\")\ntest_base = build_dataset(\"test\")\ntrain_base.write_parquet(f\"/kaggle/working/processed/train_process.parquet\")\ntest_base.write_parquet(f\"/kaggle/working/processed/test_process.parquet\")\nprint(\"\\n==============训练集数据处理+特征工程==============\")\ntrain_clean = fit_preprocess(\n    train_base,\n    output_path=\"/kaggle/working/processed/train_wide_cleaned.parquet\"\n)\nprint(\"\\n==============测试集数据处理+特征工程==============\")\ntest_clean = transform_preprocess(\n    test_base,\n    output_path=\"/kaggle/working/processed/test_wide_cleaned.parquet\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:31:24.647946Z","iopub.execute_input":"2026-06-18T13:31:24.648290Z","iopub.status.idle":"2026-06-18T13:40:22.259069Z","shell.execute_reply.started":"2026-06-18T13:31:24.648256Z","shell.execute_reply":"2026-06-18T13:40:22.258199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"训练集形状: {train_clean.shape}\")\nprint(f\"测试集形状: {test_clean.shape}\")\nprint(f\"测试集与训练集列名是否一致：{set(train_clean.columns) - {\"target\"} == set(test_clean.columns)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:41:08.549586Z","iopub.execute_input":"2026-06-18T13:41:08.550245Z","iopub.status.idle":"2026-06-18T13:41:08.559638Z","shell.execute_reply.started":"2026-06-18T13:41:08.550210Z","shell.execute_reply":"2026-06-18T13:41:08.558467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#清除多余output\nprint(\"仅保留最终训练集和测试集parquet\")\nkeep = {\n    \"/kaggle/working/processed/train_wide_cleaned.parquet\",\n    \"/kaggle/working/processed/test_wide_cleaned.parquet\",\n}\nfor p in Path(\"/kaggle/working\").rglob(\"*\"):\n    if p.is_file():\n        if str(p) not in keep:\n            try:\n                p.unlink()\n            except:\n                pass\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-18T13:41:11.790496Z","iopub.execute_input":"2026-06-18T13:41:11.790878Z","iopub.status.idle":"2026-06-18T13:41:12.141398Z","shell.execute_reply.started":"2026-06-18T13:41:11.790847Z","shell.execute_reply":"2026-06-18T13:41:12.140505Z"}},"outputs":[],"execution_count":null}]}