{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.13"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isGpuEnabled":false,"isInternetEnabled":false,"language":"python","sourceType":"notebook"},"papermill":{"default_parameters":{},"duration":400.363549,"end_time":"2026-06-18T13:48:36.641711+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-06-18T13:41:56.278162+00:00","version":"2.7.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"589bc336-2ab5-421a-9edd-c77ba0e2cd66","cell_type":"markdown","source":"# Home Credit LightGBM Kaggle Submission Notebook\n\nThis notebook integrates the preprocessing workflow with the final LightGBM model. It builds `train_wide_cleaned` and `test_wide_cleaned` from the raw competition parquet files, trains LightGBM on the full processed training set, and writes `/kaggle/working/submission.csv`.\n","metadata":{}},{"id":"e264b7c5-e700-4a7c-b2bd-e0cc2604810f","cell_type":"markdown","source":"## 1. Imports and Kaggle Paths\n","metadata":{}},{"id":"92529539-7b4c-4f51-8612-5a5c0937ba9f","cell_type":"code","source":"import os\nimport gc\nimport time\nimport warnings\nfrom pathlib import Path\n\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nfrom lightgbm import LGBMClassifier\n\nwarnings.filterwarnings(\"ignore\")\n\nprint(\"Notebook is running in:\", Path.cwd())\n","metadata":{},"outputs":[],"execution_count":null},{"id":"74eeebc6-80d4-449e-909e-f200a5ca2014","cell_type":"code","source":"# Kaggle competition data path.\n# The first path is the common Kaggle mount. The second path keeps compatibility\n# with notebooks that use the older /kaggle/input/competitions layout.\nDATA_ROOT_CANDIDATES = [\n    Path(\"/kaggle/input/home-credit-credit-risk-model-stability\"),\n    Path(\"/kaggle/input/competitions/home-credit-credit-risk-model-stability\"),\n]\n\nCOMPETITION_ROOT = next((p for p in DATA_ROOT_CANDIDATES if p.exists()), None)\nif COMPETITION_ROOT is None:\n    raise FileNotFoundError(\n        \"Cannot find Home Credit competition data under /kaggle/input. \"\n        \"Please attach the competition dataset to this Kaggle Notebook.\"\n    )\n\nDATA_PATH = COMPETITION_ROOT / \"parquet_files\"\nSAMPLE_SUBMISSION_PATH = COMPETITION_ROOT / \"sample_submission.csv\"\n\nOUTPUT_PATH = Path(\"/kaggle/working/processed\")\nOUTPUT_PATH.mkdir(parents=True, exist_ok=True)\n\nCACHE_DIR = Path(\"/kaggle/working/feature_cache\")\nCACHE_DIR.mkdir(parents=True, exist_ok=True)\n\nSUBMISSION_PATH = Path(\"/kaggle/working/submission.csv\")\n\nprint(\"COMPETITION_ROOT:\", COMPETITION_ROOT)\nprint(\"DATA_PATH:\", DATA_PATH)\nprint(\"SAMPLE_SUBMISSION_PATH exists:\", SAMPLE_SUBMISSION_PATH.exists())\n","metadata":{},"outputs":[],"execution_count":null},{"id":"396b5568-f6a1-4f88-9c1f-49798bbd0911","cell_type":"markdown","source":"## 2. Raw Data File Configuration\n","metadata":{}},{"id":"fa481862-83ea-4457-9b38-eef6e20d673b","cell_type":"code","source":"CONFIG = {\n    \"train\": {\n        \"base\": \"train_base.parquet\",\n\n        \"static_0\": [\n            \"train_static_0_0.parquet\",\n            \"train_static_0_1.parquet\"\n        ],\n\n        \"static_cb\": [\n            \"train_static_cb_0.parquet\"\n        ],\n\n        \"applprev\": [\n            \"train_applprev_1_0.parquet\",\n            \"train_applprev_1_1.parquet\"\n        ],\n        \"cba1\": [\n            \"train_credit_bureau_a_1_0.parquet\",\n            \"train_credit_bureau_a_1_1.parquet\",\n            \"train_credit_bureau_a_1_2.parquet\",\n            \"train_credit_bureau_a_1_3.parquet\"\n        ],\n        \"cbb1\": [\n            \"train_credit_bureau_b_1.parquet\"\n        ],\n        \"deposit\": [\n            \"train_deposit_1.parquet\"\n        ],\n        \"person\": [\n            \"train_person_1.parquet\"\n        ],\n        \"debit\": [\n            \"train_debitcard_1.parquet\"\n        ],\n        \"applprev2\": \"train_applprev_2.parquet\"\n    },\n    \n    \"test\": {\n        \"base\": \"test_base.parquet\",\n\n        \"static_0\": [\n            \"test_static_0_0.parquet\",\n            \"test_static_0_1.parquet\",\n            \"test_static_0_2.parquet\"\n        ],\n\n        \"static_cb\": [\n            \"test_static_cb_0.parquet\"\n        ],\n        \n        \"applprev\": [\n            \"test_applprev_1_0.parquet\",\n            \"test_applprev_1_1.parquet\",\n            \"test_applprev_1_2.parquet\"\n        ],\n        \"cba1\": [\n            \"test_credit_bureau_a_1_0.parquet\",\n            \"test_credit_bureau_a_1_1.parquet\",\n            \"test_credit_bureau_a_1_2.parquet\",\n            \"test_credit_bureau_a_1_3.parquet\",\n            \"test_credit_bureau_a_1_4.parquet\"\n        ],\n        \"cbb1\": [\n            \"test_credit_bureau_b_1.parquet\"\n        ],\n        \"deposit\": [\n            \"test_deposit_1.parquet\"\n        ],\n        \"person\": [\n            \"test_person_1.parquet\"\n        ],\n        \"debit\": [\n            \"test_debitcard_1.parquet\"\n        ],\n        \"applprev2\": \"test_applprev_2.parquet\"\n    }\n}","metadata":{},"outputs":[],"execution_count":null},{"id":"cf028c54-9653-4ef7-854c-fc0a5482a75a","cell_type":"markdown","source":"## 3. Aggregation Functions\n","metadata":{}},{"id":"7a01d099-d435-4b7c-891f-627ba73ac98e","cell_type":"code","source":"def set_table_dtypes(df):\n    if 'case_id' in df.columns:\n        df = df.with_columns(pl.col('case_id').cast(pl.Int32, strict=False))\n    return df\n\ndef load_and_merge_table(train_files,table_name,mode):\n    \"\"\"\n    加载并合并训练集多个 parquet 分片\n    \"\"\"\n    print(f\"加载 {table_name} ...\")\n    try:\n        train_df = None\n        for f in train_files:\n            full_path = DATA_PATH / mode / f\n            if not full_path.exists():\n                print(f\"    文件不存在: {full_path}\")\n                continue\n            df = pl.read_parquet(full_path)\n            if train_df is None:\n                train_df = df\n            else:\n                train_df = pl.concat([train_df, df],how=\"vertical\")\n        if train_df is None:\n            print(\"    没有成功加载任何文件\")\n            return None\n        # 仅统一 case_id 类型\n        if \"case_id\" in train_df.columns:\n            train_df = train_df.with_columns(\n                pl.col(\"case_id\").cast(pl.Int32, strict=False).alias(\"case_id\"))\n        print(f\"    合并后形状: {train_df.shape}\")\n        return train_df\n    except Exception as e:\n        print(f\"    加载失败: {e}\")\n        return None\n        \ndef apply_time_filter(df: pl.DataFrame, date_col: str, case_dates: pl.DataFrame) -> pl.DataFrame:\n    \"\"\"时间过滤：只保留 date_col < 申请日期的记录\"\"\"\n    if date_col not in df.columns:\n        return df\n    # 确保日期列是字符串格式\n    if df[date_col].dtype != pl.Utf8:\n        df = df.with_columns(pl.col(date_col).cast(pl.Utf8).alias(date_col))\n    # 确保 case_dates 中的 date_decision 也是字符串\n    case_dates_copy = case_dates.select([\n        'case_id',\n        pl.col('date_decision').cast(pl.Utf8).alias('date_decision')\n    ])\n    df = df.join(case_dates_copy, on='case_id', how='left')\n    df = df.filter(pl.col(date_col) < pl.col('date_decision'))\n    df = df.drop('date_decision')\n\n    return df\n\ndef aggregate_depth1(\n    df: pl.DataFrame,\n    group_col: str = \"case_id\",\n    prefix: str = \"\"\n):\n    agg_exprs = []\n    exclude_cols = {\n        \"case_id\",\n        \"num_group1\",\n        \"num_group2\"\n    }\n    for col in df.columns:\n        if col in exclude_cols:\n            continue\n        suffix = col[-1]\n        dtype = df.schema[col]\n        if suffix in (\"A\", \"P\"):\n            agg_exprs.extend([\n                #pl.col(col).count().alias(f\"{prefix}{col}_count\"),\n                #pl.col(col).sum().alias(f\"{prefix}{col}_sum\"),\n                pl.col(col).mean().alias(f\"{prefix}{col}_mean\"),\n                pl.col(col).max().alias(f\"{prefix}{col}_max\"),\n                pl.col(col).min().alias(f\"{prefix}{col}_min\"),\n                #pl.col(col).std().alias(f\"{prefix}{col}_std\"),\n            ])\n        elif suffix == \"L\":\n            if dtype.is_numeric():\n                agg_exprs.extend([\n                    #pl.col(col).count().alias(f\"{prefix}{col}_count\"),\n                    #pl.col(col).sum().alias(f\"{prefix}{col}_sum\"),\n                    pl.col(col).mean().alias(f\"{prefix}{col}_mean\"),\n                    pl.col(col).max().alias(f\"{prefix}{col}_max\"),\n                    pl.col(col).min().alias(f\"{prefix}{col}_min\"),\n                    #pl.col(col).std().alias(f\"{prefix}{col}_std\"),\n                ])\n            else:\n                agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        elif suffix == \"M\":\n            agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        elif suffix == \"D\":\n            agg_exprs.extend([\n                pl.col(col).max().alias(f\"{prefix}{col}_max\"),\n                pl.col(col).min().alias(f\"{prefix}{col}_min\"),\n            ])\n    result = (df.group_by(group_col).agg(agg_exprs))\n\n    return result\n\ndef process_depth1_table(file_path,table_name,prefix=\"\"):\n    print(f\"处理 {table_name}\")\n    df = pl.read_parquet(file_path)\n    if \"case_id\" in df.columns:\n        df = df.with_columns(pl.col(\"case_id\").cast(pl.Int32, strict=False))\n    agg_df = aggregate_depth1(df,group_col=\"case_id\",prefix=prefix)\n    print(f\"    原始: {df.shape} -> 聚合后: {agg_df.shape}\")\n    del df\n    gc.collect()\n\n    return agg_df\n    \ndef process_depth1_files(mode,file_list,feature_name,prefix=\"\"):\n    agg_parts = []\n    for file_name in file_list:\n        path = DATA_PATH / mode / file_name\n        if not path.exists():\n            continue\n        agg_df = process_depth1_table(path,file_name,prefix)\n        agg_parts.append(agg_df)\n    if len(agg_parts) == 0:\n        return None\n    result = pl.concat(agg_parts,how=\"vertical_relaxed\")\n    print(f\"    {feature_name} 最终形状: {result.shape}\")\n    save_feature_table(result,feature_name)\n    del agg_parts\n    gc.collect()\n\n    return result\n\ndef save_feature_table(df,feature_name):\n    path = CACHE_DIR / f\"{feature_name}.parquet\"\n    df.write_parquet(path)\n    print(f\"    保存成功: {path}\")\n    \n    return path\n\ndef load_feature_table(feature_name):\n    path = CACHE_DIR / f\"{feature_name}.parquet\"\n    \n    return pl.read_parquet(path)\n\ndef aggregate_depth2_level1(df,prefix: \"applprev2\"):\n    agg_exprs = []\n    exclude_cols = {\n        \"case_id\",\n        \"num_group1\",\n        \"num_group2\"\n    }\n    for col in df.columns:\n        if col in exclude_cols:\n            continue\n        suffix = col[-1]\n        dtype = df.schema[col]\n        # A/P\n        if suffix in (\"A\", \"P\"):\n            agg_exprs.extend([\n                #pl.col(col).count().alias(f\"{col}_count\"),\n                #pl.col(col).sum().alias(f\"{col}_sum\"),\n                pl.col(col).mean().alias(f\"{col}_mean\"),\n                pl.col(col).max().alias(f\"{col}_max\"),\n                pl.col(col).min().alias(f\"{col}_min\"),\n                #pl.col(col).std().alias(f\"{col}_std\"),\n            ])\n        # L 自动识别\n        elif suffix == \"L\":\n            if dtype.is_numeric():\n                agg_exprs.extend([\n                    #pl.col(col).count().alias(f\"{col}_count\"),\n                    #pl.col(col).sum().alias(f\"{col}_sum\"),\n                    pl.col(col).mean().alias(f\"{col}_mean\"),\n                    pl.col(col).max().alias(f\"{col}_max\"),\n                    pl.col(col).min().alias(f\"{col}_min\"),\n                    #pl.col(col).std().alias(f\"{col}_std\"),\n                ])\n            else:\n                agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        # M\n        elif suffix == \"M\":\n            agg_exprs.append(pl.col(col).mode().first().alias(f\"{prefix}{col}_mode\"))\n        # D\n        elif suffix == \"D\":\n            agg_exprs.extend([\n                pl.col(col).max().alias(f\"{col}_max\"),\n                pl.col(col).min().alias(f\"{col}_min\"),\n            ])\n            \n    return (df.group_by([\"case_id\", \"num_group1\"]).agg(agg_exprs))\n    \ndef aggregate_depth2_level2(df):\n    agg_cols = [c for c in df.columns if c not in (\"case_id\", \"num_group1\")]\n    agg_exprs = []\n    for col in agg_cols:\n        agg_exprs.extend([\n            pl.col(col).mean().alias(f\"{col}_mean_across_contracts\"),\n            pl.col(col).max().alias(f\"{col}_max_across_contracts\"),\n            pl.col(col).sum().alias(f\"{col}_sum_across_contracts\"),\n            pl.col(col).std().alias(f\"{col}_std_across_contracts\"),\n        ])\n        \n    return (df.group_by(\"case_id\").agg(agg_exprs))\n\ndef process_applprev2(path,case_dates):\n    print(\"处理 applprev_2\")\n    df = pl.read_parquet(path)\n    print(\"原始:\", df.shape)\n    # 时间过滤\n    if \"creationdate_885D\" in df.columns:\n        df = apply_time_filter(df,\"creationdate_885D\",case_dates)\n        print(\"时间过滤后:\", df.shape)\n    level1 = aggregate_depth2_level1(df,prefix=\"applprev2\")\n    print(\"第一层聚合:\",level1.shape)\n    save_feature_table(level1,\"applprev2_level1\")\n    del df\n    del level1\n    gc.collect()\n    level1 = load_feature_table(\"applprev2_level1\")\n    level2 = aggregate_depth2_level2(level1)\n    print(\"第二层聚合:\",level2.shape)\n    save_feature_table(level2,\"applprev2_final\")\n    del level1\n    gc.collect()\n\n    return None\n\n#数据聚合\ndef build_dataset(mode=\"train\"):\n    ##base表\n    cfg = CONFIG[mode]\n    base_file = cfg[\"base\"]\n    base_file = pl.read_parquet(DATA_PATH / mode / base_file)\n    print(f\"base表形状: {base_file.shape}\")\n    cols = [\n        pl.col(\"case_id\").cast(pl.Int32, strict=False),\n        pl.col(\"WEEK_NUM\").cast(pl.Int32, strict=False),\n        pl.col(\"date_decision\").cast(pl.Date, strict=False),\n    ]\n    if \"target\" in base_file.columns:\n        cols.append(pl.col(\"target\").cast(pl.Int8, strict=False))\n    base = base_file.with_columns(cols)\n    key_cols = [\n        \"case_id\",\n        \"target\",\n        \"date_decision\",\n        \"WEEK_NUM\"\n    ]\n    base = base.select([c for c in key_cols if c in base.columns])\n    base = base.select([c for c in key_cols if c in base.columns])\n    print(f\"base表形状: {base.shape}\")\n    case_dates = base.select([\"case_id\", \"date_decision\"])\n    del base_file\n    gc.collect()\n    \n    ##depth=0\n    static_0 = load_and_merge_table(cfg[\"static_0\"],\"static_0\",mode)\n    base = base.join(static_0,on=\"case_id\",how=\"left\")\n    print(f\"    新增 {static_0.shape[1] - 1} 列\")\n    del static_0\n    gc.collect()\n    static_cb = load_and_merge_table(cfg[\"static_cb\"],\"static_cb_0\",mode)\n    base = base.join(static_cb,on=\"case_id\",how=\"left\")\n    print(f\"    新增 {static_cb.shape[1] - 1} 列\")\n    del static_cb\n    gc.collect()\n    print(f\"    当前base表列数: {base.shape[1]}\")\n    print(f\"    当前base表行数: {base.shape[0]}\")\n    \n    ##depth=1\n    DEPTH1_TABLES = {k: cfg[k] for k in [\"applprev\",\"cba1\",\"cbb1\",\"deposit\",\"person\",\"debit\"]}\n    for prefix, files in DEPTH1_TABLES.items():\n        print(f\"处理 {prefix}\")\n        agg_df = process_depth1_files(mode,files,feature_name=prefix,prefix=f\"{prefix}_\")\n        if agg_df is None:\n            continue\n        # 保存聚合结果\n        agg_path = CACHE_DIR / f\"{prefix}_agg.parquet\"\n        agg_df.write_parquet(agg_path)\n        print(f\"    聚合结果已保存: {agg_path}\")\n        print(f\"    聚合结果形状: {agg_df.shape}\")\n        before_cols = base.shape[1]\n        #base\n        base = base.join(agg_df,on=\"case_id\",how=\"left\")\n        after_cols = base.shape[1]\n        print(f\"    新增 {after_cols - before_cols} 列\")\n        print(f\"    当前 base: {base.shape}\")\n        del agg_df\n        gc.collect()\n        base_path = CACHE_DIR / f\"base_after_{prefix}.parquet\"\n        base.write_parquet(base_path)\n        print(f\"    当前base已保存: {base_path}\")\n        del base\n        gc.collect()\n        base = pl.read_parquet(base_path)\n        print(f\"重新加载base: {base.shape}\")\n    \n    ##depth=2\n    path = DATA_PATH / mode / cfg[\"applprev2\"]\n    process_applprev2(path,case_dates)\n    applprev2 = load_feature_table(\"applprev2_final\")\n    rename_map = {c: f\"applprev2_{c}\" for c in applprev2.columns if c != \"case_id\"}\n    applprev2 = applprev2.rename(rename_map)\n    #base\n    before_cols = base.shape[1]\n    base = base.join(applprev2,on=\"case_id\",how=\"left\")\n    after_cols = base.shape[1]\n    print(f\"    新增 {after_cols-before_cols} 列\")\n    save_feature_table(base,\"base_after_applprev2\")\n    del applprev2\n    gc.collect()\n\n    return base","metadata":{},"outputs":[],"execution_count":null},{"id":"ec1f232d-38f7-4a12-849e-21f5c52fccd7","cell_type":"markdown","source":"## 4. Cleaning and Feature Engineering Functions\n","metadata":{}},{"id":"97161525-531f-4e6c-ade0-247f3be2a35a","cell_type":"code","source":"def clean_missing_and_constant_features(\n    df,\n    missing_threshold=0.90,\n    save_report=True\n):\n    \"\"\"\n    删除：\n    1. 全空列\n    2. 高缺失列（默认>90%）\n    3. 单值列\n    4. 单值+空值列\n\n    返回：\n    cleaned_df\n    report_df\n    \"\"\"\n    n_rows = len(df)\n    drop_cols = []\n    report = []\n    for col in df.columns:\n        if col in [\"case_id\", \"target\"]:\n            continue\n        s = df[col]\n        null_ratio = s.null_count() / n_rows\n        try:\n            nunique = s.drop_nulls().n_unique()\n        except:\n            nunique = None\n        reason = None\n        if null_ratio == 1:\n            reason = \"all_null\"\n        elif null_ratio > missing_threshold:\n            reason = \"high_missing\"\n        elif nunique == 1 and null_ratio == 0:\n            reason = \"constant\"\n        elif nunique <= 1:\n            reason = \"constant_plus_null\"\n        if reason is not None:\n            drop_cols.append(col)\n            report.append({\n                \"column\": col,\n                \"reason\": reason,\n                \"null_ratio\": round(null_ratio, 6),\n                \"nunique\": nunique,\n                \"dtype\": str(df.schema[col])\n            })    \n    print(f\"低质数据处理-待删除列数: {len(drop_cols)}\")\n    cleaned_df = df.drop(drop_cols)\n    print(f\"    处理前: {df.shape}\")\n    print(f\"    处理后: {cleaned_df.shape}\")\n    report_df = pl.DataFrame(report)\n    if save_report:\n        report_df.write_csv(\"drop_feature_report.csv\")\n        print(\"    删除报告已保存: drop_feature_report.csv\")\n\n    return cleaned_df, report_df, drop_cols\n\ndef convert_dates_to_days(df, decision_col=\"date_decision\"):\n    date_cols = [\n        c\n        for c in df.columns\n        if c.endswith(\"D\")\n        and c != decision_col\n    ]\n    print(f\"日期数据处理-发现 {len(date_cols)} 个日期列\")\n    for col in date_cols:\n        print(f\"    处理 {col}\")\n        dtype = df.schema[col]\n        # 已经是 Date\n        if dtype == pl.Date:\n            date_expr = pl.col(col)\n        else:\n            date_expr = (\n                pl.col(col)\n                .cast(pl.String, strict=False)\n                .str.to_date(strict=False)\n            )\n        df = df.with_columns([\n            pl.col(col).is_null().cast(pl.Int8).alias(f\"{col}_missing\"),\n            (date_expr- pl.col(decision_col)).dt.total_days().alias(f\"{col}_days\")\n        ])\n    drop_cols = [c for c in date_cols + [decision_col] if c in df.columns]\n    df = df.drop(drop_cols)\n    print(f\"    删除 {len(drop_cols)} 个原始日期列\")\n\n    return df\n\ndef export_category_summary(df):\n    rows = []\n    cat_cols = []\n    for col in df.columns:\n        dtype = df.schema[col]\n        if dtype == pl.String:\n            cat_cols.append(col)\n            rows.append({\n                \"column\": col,\n                \"nunique\": (df[col].drop_nulls().n_unique()),\n                \"missing_ratio\": (df[col].null_count() / len(df))\n            })\n    summary = (pl.DataFrame(rows).sort(\"nunique\"))\n    summary.write_csv(\"category_summary.csv\")\n    print(f\"类别数据处理-查找到类别列数: {len(cat_cols)}\")\n\n    return summary\n\ndef one_hot_low_cardinality(\n    df,\n    threshold=15,\n    columns=None\n):\n    if columns is None:\n        cat_cols = []\n        for col in df.columns:\n            if df.schema[col] != pl.String:\n                continue\n            nunique = df[col].drop_nulls().n_unique()\n            if nunique <= threshold:\n                cat_cols.append(col)\n    else:\n        cat_cols = [c for c in columns if c in df.columns]\n    print(f\"OHE列数: {len(cat_cols)}\")\n    if len(cat_cols) == 0:\n        return df, cat_cols\n    df = df.to_dummies(columns=cat_cols)\n    return df, cat_cols\n   \ndef drop_high_cardinality_ids(\n    df,\n    unique_threshold=50,\n    columns=None\n):\n    if columns is None:\n        drop_cols = []\n        for col in df.columns:\n            if df.schema[col] != pl.String:\n                continue\n            nunique = df[col].drop_nulls().n_unique()\n            if nunique > unique_threshold:\n                drop_cols.append(col)\n    else:\n        drop_cols = [c for c in columns if c in df.columns]\n    print(f\"删除高基数列: {len(drop_cols)}\")\n    return df.drop(drop_cols), drop_cols\n\ndef create_missing_flags(\n    df,\n    threshold=0.3,\n    columns=None\n):\n    if columns is None:\n        prefixes = (\n            \"cba1_\",\n            \"cbb1_\",\n            \"deposit_\",\n            \"debit_\",\n            \"person_\",\n            \"applprev_\",\n            \"applprev2_\"\n        )\n        suffixes = (\n            \"_mean\",\n            \"_sum\",\n            \"_max\",\n            \"_min\",\n            \"_std\"\n        )\n        flag_cols = []\n        for col in df.columns:\n            if not col.startswith(prefixes):\n                continue\n            if not any(s in col for s in suffixes):\n                continue\n            miss_ratio = (df[col].null_count() / len(df))\n            if miss_ratio > threshold:\n                flag_cols.append(col)\n    else:\n        flag_cols = [c for c in columns if c in df.columns]\n    new_cols = []\n    for col in flag_cols:\n        new_cols.append(pl.col(col).is_null().cast(pl.Int8).alias(f\"{col}_missing\"))\n    if len(new_cols):\n        df = df.with_columns(new_cols)\n    print(f\"新增 Missing Flag: {len(new_cols)}\")\n\n    return df, flag_cols\n\ndef compress_dtypes(df):\n    for col in df.columns:\n        dtype = df.schema[col]\n        if dtype == pl.Float64:\n            df = df.with_columns(pl.col(col).cast(pl.Float32))\n        elif dtype == pl.Int64:\n            df = df.with_columns(pl.col(col).cast(pl.Int32))\n\n    return df\n\ndef corr_filter_by_group(df,profile,corr_threshold=0.98):\n    remove_cols = set()\n    numeric_types = (\n        pl.Int8,\n        pl.Int16,\n        pl.Int32,\n        pl.Int64,\n        pl.Float32,\n        pl.Float64\n    )\n    numeric_cols = [\n        c\n        for c in df.columns\n        if df.schema[c] in numeric_types\n        and c not in [\"case_id\", \"target\"]\n    ]\n    groups = (profile[\"missing_group\"].unique().sort().to_list())\n    \n    for group in groups:\n        print(f\"group {group}\")\n        cols = (profile.filter(pl.col(\"missing_group\") == group)[\"column\"].to_list())\n        cols = [c for c in cols if c in numeric_cols]\n        if len(cols) < 2:\n            continue\n        print(f\"    group内特征数: {len(cols)}\")\n        # 只取当前group\n        tmp = (df.select(cols).to_pandas())\n        corr = tmp.corr().abs()\n        upper = corr.where(np.triu(np.ones(corr.shape),k=1).astype(bool))\n        for col in upper.columns:\n            high_corr = upper.index[upper[col] > corr_threshold]\n            for other in high_corr:\n                nunique_col = (profile.filter(pl.col(\"column\") == col)[\"nunique\"][0])\n                nunique_other = (profile.filter(pl.col(\"column\") == other)[\"nunique\"][0])\n                if nunique_col >= nunique_other:\n                    remove_cols.add(other)\n                else:\n                    remove_cols.add(col)\n        print(f\"    累计删除: {len(remove_cols)}\")\n        del tmp\n        del corr\n        del upper\n        gc.collect()\n        \n    return list(remove_cols)\n\ndef build_feature_profile(df):\n    rows = []\n    n = len(df)\n    for col in df.columns:\n        if col in [\"case_id\", \"target\"]:\n            continue\n        rows.append({\n            \"column\": col,\n            \"missing_ratio\": df[col].null_count() / n,\n            \"nunique\": (df[col].drop_nulls().n_unique()),\n            \"dtype\": str(df.schema[col])\n        })\n    profile = pl.DataFrame(rows)\n    profile.write_csv(\"feature_profile.csv\")\n    print(profile.shape)\n\n    return profile\n\ndef add_missing_group(profile):\n    profile = profile.with_columns((\n            (pl.col(\"missing_ratio\") * 20)\n            .floor()\n            .cast(pl.Int32)\n        ).alias(\"missing_group\")\n    )\n\n    return profile\n\ndef align_test_columns(test_df,train_columns):\n    for col in train_columns:\n        if col not in test_df.columns:\n            test_df = test_df.with_columns(pl.lit(None).alias(col))\n\n    return test_df.select(train_columns)\n","metadata":{},"outputs":[],"execution_count":null},{"id":"10da6183-ef40-4416-bc03-c5de1df272d2","cell_type":"code","source":"#fit for train\ndef fit_preprocess(\n    base,\n    output_path=None,\n    meta_path=\"/kaggle/working/preprocess_meta.pkl\"\n):\n    metadata = {}\n    print(\"===== Step1: Clean =====\")\n    base, drop_report, drop_cols = (\n        clean_missing_and_constant_features(\n            base,\n            missing_threshold=0.90\n        )\n    )\n    metadata[\"drop_cols\"] = drop_cols\n    \n    print(\"===== Step2: Dates =====\")\n    date_cols = [\n        c\n        for c in base.columns\n        if c.endswith(\"D\")\n        and c != \"date_decision\"\n    ]\n    metadata[\"date_cols\"] = date_cols\n    base = convert_dates_to_days(base)\n    \n    print(\"===== Step3: OHE =====\")\n    base, ohe_cols = one_hot_low_cardinality(base,threshold=15)\n    metadata[\"ohe_cols\"] = ohe_cols\n    \n    print(\"===== Step4: High Cardinality =====\")\n    base, high_card_cols = (drop_high_cardinality_ids(base,unique_threshold=50))\n    metadata[\"high_card_cols\"] = high_card_cols\n    \n    print(\"===== Step5: Missing Flags =====\")\n    base, missing_flag_cols = (create_missing_flags(base,threshold=0.30))\n    metadata[\"missing_flag_cols\"] = missing_flag_cols\n    \n    print(\"===== Step6: Compress =====\")\n    base = compress_dtypes(base)\n    \n    print(\"===== Step7: Correlation =====\")\n    feature_profile = build_feature_profile(base)\n    feature_profile = add_missing_group(feature_profile)\n    corr_sample = base.sample(n=min(300000, len(base)),seed=42)\n    drop_corr_cols = corr_filter_by_group(\n        corr_sample,\n        feature_profile,\n        corr_threshold=0.98\n    )\n    metadata[\"drop_corr_cols\"] = drop_corr_cols\n    base = base.drop([c for c in drop_corr_cols if c in base.columns])\n    del corr_sample, feature_profile\n    gc.collect()\n    metadata[\"final_columns\"] = base.columns\n    print(f\"最终训练集形状: {base.shape}\")\n    joblib.dump(\n        metadata,\n        meta_path\n    )\n    print(f\"metadata已保存: {meta_path}\")\n    if output_path:\n        base.write_parquet(output_path)\n\n    return base\n\n#transform for test\ndef transform_preprocess(\n    base,\n    meta_path=\"/kaggle/working/preprocess_meta.pkl\",\n    output_path=None\n):\n    metadata = joblib.load(meta_path)\n    print(\"===== Step1: Drop Training Removed Columns =====\")\n    base = base.drop([c for c in metadata[\"drop_cols\"] if c in base.columns])\n\n    print(\"===== Step2: Dates =====\")\n    base = convert_dates_to_days(base)\n\n    print(\"===== Step3: OHE =====\")\n    base, _ = one_hot_low_cardinality(\n        base,\n        columns=metadata[\"ohe_cols\"]\n    )\n\n    print(\"===== Step4: High Cardinality =====\")\n    base, _ = drop_high_cardinality_ids(\n        base,\n        columns=metadata[\"high_card_cols\"]\n    )\n\n    print(\"===== Step5: Missing Flags =====\")\n    base, _ = create_missing_flags(\n        base,\n        columns=metadata[\"missing_flag_cols\"]\n    )\n    print(\"===== Step6: Compress =====\")\n    base = compress_dtypes(base)\n    \n    print(\"===== Step7: Correlation Columns =====\")\n    base = base.drop([c for c in metadata[\"drop_corr_cols\"] if c in base.columns])\n    \n    print(\"===== Step8: Align Columns =====\")\n    train_cols = metadata[\"final_columns\"]\n    for col in train_cols:\n        if col not in base.columns:\n            base = base.with_columns(\n                pl.lit(None).alias(col)\n            )\n    base = base.select(train_cols)\n    base = base.drop('target')\n    print(f\"最终测试集形状: {base.shape}\")\n    if output_path:\n        base.write_parquet(output_path)\n\n    return base","metadata":{},"outputs":[],"execution_count":null},{"id":"6f22fa3f-9279-4a26-9ff6-a3d77541f14c","cell_type":"markdown","source":"## 5. Build Processed Train/Test Tables\n","metadata":{}},{"id":"bf66b631-1be8-4c85-a648-facfc4a2895b","cell_type":"code","source":"\nprint(\"=================== 训练集聚合 ===================\")\ntrain_base = build_dataset(\"train\")\n\nprint(\"\\n============== 训练集数据处理 + 特征工程 ==============\")\ntrain_clean = fit_preprocess(\n    train_base,\n    output_path=OUTPUT_PATH / \"train_wide_cleaned.parquet\",\n    meta_path=\"/kaggle/working/preprocess_meta.pkl\",\n)\ndel train_base\ngc.collect()\n\nprint(\"\\n=================== 测试集聚合 ===================\")\ntest_base = build_dataset(\"test\")\n\nprint(\"\\n============== 测试集数据处理 + 特征工程 ==============\")\ntest_clean = transform_preprocess(\n    test_base,\n    meta_path=\"/kaggle/working/preprocess_meta.pkl\",\n    output_path=OUTPUT_PATH / \"test_wide_cleaned.parquet\",\n)\ndel test_base\ngc.collect()\n\nprint(\"训练集形状:\", train_clean.shape)\nprint(\"测试集形状:\", test_clean.shape)\nprint(\"测试集与训练集列名是否一致:\", set(train_clean.columns) - {\"target\"} == set(test_clean.columns))\n","metadata":{},"outputs":[],"execution_count":null},{"id":"98d17734-df85-46af-9b10-eb1d2f284668","cell_type":"markdown","source":"## 6. Train Final LightGBM Model\n","metadata":{}},{"id":"e0f555bb-45e9-4b85-9940-4682f52c0be3","cell_type":"code","source":"\n# Final LightGBM model parameters selected from local validation experiments.\n# This Kaggle submission version is memory-safe for reruns: it caps training rows\n# and converts feature matrices to float32 numpy arrays before fitting.\nID_COL = \"case_id\"\nTARGET_COL = \"target\"\nRANDOM_STATE = 42\nMAX_TRAIN_ROWS = 900_000\n\nBEST_PARAMS = {\n    \"objective\": \"binary\",\n    \"n_estimators\": 1200,\n    \"learning_rate\": 0.03,\n    \"num_leaves\": 31,\n    \"min_child_samples\": 300,\n    \"subsample\": 0.7,\n    \"colsample_bytree\": 0.7,\n    \"reg_lambda\": 10.0,\n    \"random_state\": RANDOM_STATE,\n    \"n_jobs\": -1,\n    \"verbosity\": -1,\n}\n\nprint(\"Converting processed Polars frames to pandas for LightGBM...\")\ntrain_df = train_clean.to_pandas()\ndel train_clean\ngc.collect()\n\ntest_df = test_clean.to_pandas()\ndel test_clean\ngc.collect()\n\ndef needs_encoding(s):\n    return (\n        pd.api.types.is_object_dtype(s)\n        or pd.api.types.is_string_dtype(s)\n        or pd.api.types.is_categorical_dtype(s)\n    )\n\nobject_cols = [\n    c for c in train_df.columns\n    if c not in [ID_COL, TARGET_COL] and needs_encoding(train_df[c])\n]\nobject_cols += [\n    c for c in test_df.columns\n    if c not in [ID_COL, TARGET_COL] and needs_encoding(test_df[c]) and c not in object_cols\n]\n\nprint(\"Columns to encode:\", len(object_cols))\nif object_cols:\n    print(object_cols[:30])\n\nfor col in object_cols:\n    train_s = train_df[col].astype(\"string\") if col in train_df.columns else pd.Series([], dtype=\"string\")\n    test_s = test_df[col].astype(\"string\") if col in test_df.columns else pd.Series([], dtype=\"string\")\n    categories = pd.Index(pd.concat([train_s, test_s], ignore_index=True).dropna().unique())\n    mapping = {value: idx for idx, value in enumerate(categories)}\n\n    if col in train_df.columns:\n        train_df[col] = train_s.map(mapping).fillna(-1).astype(\"int32\")\n    if col in test_df.columns:\n        test_df[col] = test_s.map(mapping).fillna(-1).astype(\"int32\")\n    del train_s, test_s, categories, mapping\n    gc.collect()\n\nfeature_cols = [c for c in train_df.columns if c not in [ID_COL, TARGET_COL]]\nmissing_in_test = sorted(set(feature_cols) - set(test_df.columns))\nextra_in_test = sorted(set(test_df.columns) - set(feature_cols) - {ID_COL})\n\nif missing_in_test:\n    raise ValueError(f\"Test set is missing feature columns: {missing_in_test[:10]}\")\nif extra_in_test:\n    print(\"Extra columns in test ignored:\", extra_in_test[:10])\n\nif len(train_df) > MAX_TRAIN_ROWS:\n    frac = MAX_TRAIN_ROWS / len(train_df)\n    print(f\"Sampling train data from {len(train_df)} to about {MAX_TRAIN_ROWS} rows, stratified by target.\")\n    train_df = (\n        train_df\n        .groupby(TARGET_COL, group_keys=False)\n        .sample(frac=frac, random_state=RANDOM_STATE)\n        .reset_index(drop=True)\n    )\n    gc.collect()\n\ny_train = train_df[TARGET_COL].astype(\"int8\").to_numpy()\nX_train = train_df[feature_cols].astype(\"float32\").to_numpy()\ntest_ids = test_df[ID_COL].to_numpy()\nX_test = test_df[feature_cols].astype(\"float32\").to_numpy()\n\ndel train_df, test_df\ngc.collect()\n\nprint(\"X_train:\", X_train.shape, X_train.dtype)\nprint(\"X_test:\", X_test.shape, X_test.dtype)\nprint(\"Target rate:\", y_train.mean())\n\nfinal_model = LGBMClassifier(**BEST_PARAMS)\n\nstart = time.time()\nfinal_model.fit(X_train, y_train)\nelapsed = time.time() - start\n\nprint(f\"Final model trained in {elapsed:.1f} seconds\")\n","metadata":{},"outputs":[],"execution_count":null},{"id":"50957494-8190-491d-8c83-d3283ec26a7f","cell_type":"markdown","source":"## 7. Generate Kaggle Submission\n","metadata":{}},{"id":"fd6575b7-1d9c-49c1-b046-502fe5fdb801","cell_type":"code","source":"\ntest_pred = final_model.predict_proba(X_test)[:, 1]\n\ndel X_train, y_train, X_test, final_model\ngc.collect()\n\nprediction_df = pd.DataFrame({\n    ID_COL: test_ids,\n    \"score\": test_pred,\n})\n\nif SAMPLE_SUBMISSION_PATH.exists():\n    sample_submission = pd.read_csv(SAMPLE_SUBMISSION_PATH)\n    submission = sample_submission[[ID_COL]].merge(prediction_df, on=ID_COL, how=\"left\")\n    if submission[\"score\"].isna().any():\n        fill_value = float(prediction_df[\"score\"].mean())\n        print(f\"Warning: filling missing scores with mean prediction {fill_value:.6f}\")\n        submission[\"score\"] = submission[\"score\"].fillna(fill_value)\nelse:\n    submission = prediction_df\n\nsubmission.to_csv(SUBMISSION_PATH, index=False)\n\nprint(\"Saved submission to:\", SUBMISSION_PATH)\nprint(submission.shape)\nsubmission.head()\n","metadata":{},"outputs":[],"execution_count":null},{"id":"e9f648cd-da73-46aa-9f0b-ad63736a6385","cell_type":"markdown","source":"## 8. Cleanup Output Files\n","metadata":{}},{"id":"d12e4890-06c6-4b3a-9183-c32d52b1318f","cell_type":"code","source":"# Optional cleanup to keep Kaggle Notebook output small.\n# Keep only submission.csv and remove large intermediate files after submission is created.\nif SUBMISSION_PATH.exists() and Path(\"/kaggle/working\").exists():\n    keep = {SUBMISSION_PATH.resolve()}\n    for p in Path(\"/kaggle/working\").rglob(\"*\"):\n        if p.is_file():\n            try:\n                if p.resolve() not in keep:\n                    p.unlink()\n            except Exception as exc:\n                print(\"Skip cleanup for\", p, \"because\", exc)\n    gc.collect()\n    print(\"Cleanup complete. Kept:\", SUBMISSION_PATH)\nelse:\n    print(\"Skip cleanup because submission.csv does not exist.\")\n","metadata":{},"outputs":[],"execution_count":null}]}