{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":9509911,"sourceType":"datasetVersion","datasetId":5738294},{"sourceId":9528826,"sourceType":"datasetVersion","datasetId":5802772}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\n\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\nimport joblib\nimport lightgbm as lgb\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:33.691752Z","iopub.execute_input":"2024-10-12T14:35:33.692466Z","iopub.status.idle":"2024-10-12T14:35:37.006416Z","shell.execute_reply.started":"2024-10-12T14:35:33.692404Z","shell.execute_reply":"2024-10-12T14:35:37.004982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 导入数据","metadata":{}},{"cell_type":"code","source":"#数据所在地址\nROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.008952Z","iopub.execute_input":"2024-10-12T14:35:37.009653Z","iopub.status.idle":"2024-10-12T14:35:37.016643Z","shell.execute_reply.started":"2024-10-12T14:35:37.009607Z","shell.execute_reply":"2024-10-12T14:35:37.014852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#读取定义文件（原始数据变量汇总）\nfeature_definitions = pd.read_csv(\"/kaggle/input/feature-definitions/feature_definitions.csv\")\nfeature_definitions","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.034823Z","iopub.execute_input":"2024-10-12T14:35:37.035537Z","iopub.status.idle":"2024-10-12T14:35:37.081648Z","shell.execute_reply.started":"2024-10-12T14:35:37.035476Z","shell.execute_reply":"2024-10-12T14:35:37.080281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#读取更新后定义文件（思毅筛选后的变量汇总：last_sex_738L被删去，剩余199个变量）\ndata_train_df_feature_all = pd.read_csv(\"/kaggle/input/feature-definitions/data_train_df_feature_all.csv\")\ndata_train_df_feature_all","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.083266Z","iopub.execute_input":"2024-10-12T14:35:37.083654Z","iopub.status.idle":"2024-10-12T14:35:37.107789Z","shell.execute_reply.started":"2024-10-12T14:35:37.083614Z","shell.execute_reply":"2024-10-12T14:35:37.106381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df = data_train_df_feature_all[data_train_df_feature_all['Column Name'].str.endswith('D')]\nfiltered_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.109440Z","iopub.execute_input":"2024-10-12T14:35:37.109831Z","iopub.status.idle":"2024-10-12T14:35:37.128906Z","shell.execute_reply.started":"2024-10-12T14:35:37.109790Z","shell.execute_reply":"2024-10-12T14:35:37.127468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = feature_definitions[feature_definitions['Variable'] == 'date_decision']\nrow","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.130611Z","iopub.execute_input":"2024-10-12T14:35:37.131213Z","iopub.status.idle":"2024-10-12T14:35:37.145750Z","shell.execute_reply.started":"2024-10-12T14:35:37.131144Z","shell.execute_reply":"2024-10-12T14:35:37.144436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 对数据做类型转换和筛选","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n    #设置表格数据类型\n    @staticmethod\n    def set_table_dtypes(df): #Standardize the dtype.\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    #处理日期，将D的特性更改为date_decision的天数差。\n    @staticmethod\n    def handle_dates(df): \n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n#过滤的方法2\n    @staticmethod\n    def filter_cols(df): \n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                if col[-1] in (\"P\",\"A\") :\n                    #连续的空值大于0.3删除\n                    isnull = df[col].is_null().mean()\n                    if isnull > 0.30:\n                        df = df.drop(col)\n                else:\n                     #其他类型空值大于0.5删除\n                    isnull = df[col].is_null().mean()\n                    if isnull > 0.50:\n                        df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                #后续特征封箱freq大于200可不用删除\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.147721Z","iopub.execute_input":"2024-10-12T14:35:37.148208Z","iopub.status.idle":"2024-10-12T14:35:37.168418Z","shell.execute_reply.started":"2024-10-12T14:35:37.148164Z","shell.execute_reply":"2024-10-12T14:35:37.167085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 自动聚合\n定义了一个名为Aggregator的类，其中包含多个静态方法，用于生成聚合表达式","metadata":{}},{"cell_type":"code","source":"\nclass Aggregator:\n    \n    #为数值类型的列生成聚合表达式\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        return expr_max +expr_mean\n    \n    #为日期类型的列生成聚合表达式\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        return  expr_max +expr_mean\n    \n    #为字符串类型的列生成聚合表达式\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        return  expr_max +expr_last#+expr_count\n    \n    #为未指定类型的列生成聚合表达式\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        return  expr_max +expr_last\n    \n    #提取每个num_group的最大值和最小值\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] \n        expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        return  expr_max +expr_min\n    \n    #执行上面的函数并返回结果\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.170593Z","iopub.execute_input":"2024-10-12T14:35:37.171201Z","iopub.status.idle":"2024-10-12T14:35:37.191536Z","shell.execute_reply.started":"2024-10-12T14:35:37.171142Z","shell.execute_reply":"2024-10-12T14:35:37.190130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取文件的函数","metadata":{}},{"cell_type":"code","source":"#对于单个文件读取\ndef read_file(path, depth=None): \n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n     #·分组聚合·->·多条case·id的记录 按照 最大值 进行合并\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg( Aggregator.get_exprs(df))\n    \n    return df\n\n#对于多个文件读取\ndef read_files(regex_path, depth=None):\n    chunks = []\n    #glob用于通配符，读取之后最后的_0_1不一样的那些文件\n    for path in glob(str(regex_path)):\n        chunks.append(pl.read_parquet(path).pipe(Pipeline.set_table_dtypes))\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\") #拼接的操作concat，按行\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg( Aggregator.get_exprs(df))\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.503208Z","iopub.execute_input":"2024-10-12T14:35:37.503650Z","iopub.status.idle":"2024-10-12T14:35:37.513611Z","shell.execute_reply.started":"2024-10-12T14:35:37.503610Z","shell.execute_reply":"2024-10-12T14:35:37.512330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 特征工程函数，用于添加新特征和合并数据框","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    \n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:37.926295Z","iopub.execute_input":"2024-10-12T14:35:37.926740Z","iopub.status.idle":"2024-10-12T14:35:37.935380Z","shell.execute_reply.started":"2024-10-12T14:35:37.926700Z","shell.execute_reply":"2024-10-12T14:35:37.933998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#转换成dataframe的数据格式\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:38.168850Z","iopub.execute_input":"2024-10-12T14:35:38.170105Z","iopub.status.idle":"2024-10-12T14:35:38.177014Z","shell.execute_reply.started":"2024-10-12T14:35:38.170004Z","shell.execute_reply":"2024-10-12T14:35:38.175604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取训练集数据和特征工程","metadata":{}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:35:39.026667Z","iopub.execute_input":"2024-10-12T14:35:39.027167Z","iopub.status.idle":"2024-10-12T14:36:12.272232Z","shell.execute_reply.started":"2024-10-12T14:35:39.027120Z","shell.execute_reply":"2024-10-12T14:36:12.270583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:12.274372Z","iopub.execute_input":"2024-10-12T14:36:12.274794Z","iopub.status.idle":"2024-10-12T14:36:23.601418Z","shell.execute_reply.started":"2024-10-12T14:36:12.274750Z","shell.execute_reply":"2024-10-12T14:36:23.600091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#筛选数据（字符串类别和大量空值数据）\ndf_train = df_train.pipe(Pipeline.filter_cols)\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:23.602764Z","iopub.execute_input":"2024-10-12T14:36:23.603193Z","iopub.status.idle":"2024-10-12T14:36:26.888955Z","shell.execute_reply.started":"2024-10-12T14:36:23.603143Z","shell.execute_reply":"2024-10-12T14:36:26.887548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:26.892613Z","iopub.execute_input":"2024-10-12T14:36:26.893181Z","iopub.status.idle":"2024-10-12T14:36:26.920469Z","shell.execute_reply.started":"2024-10-12T14:36:26.893109Z","shell.execute_reply":"2024-10-12T14:36:26.919122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#减少内存函数\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:26.922138Z","iopub.execute_input":"2024-10-12T14:36:26.922611Z","iopub.status.idle":"2024-10-12T14:36:26.941470Z","shell.execute_reply.started":"2024-10-12T14:36:26.922554Z","shell.execute_reply":"2024-10-12T14:36:26.940197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()\ndf_train, cat_cols = to_pandas(df_train)\ndf_train = reduce_mem_usage(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:26.943295Z","iopub.execute_input":"2024-10-12T14:36:26.943688Z","iopub.status.idle":"2024-10-12T14:36:51.853928Z","shell.execute_reply.started":"2024-10-12T14:36:26.943648Z","shell.execute_reply":"2024-10-12T14:36:51.852513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 特征筛选——增加和删除变量","metadata":{}},{"cell_type":"markdown","source":"1. 申请时的客户年龄（后续分析年龄分布与逾期情况）","metadata":{}},{"cell_type":"code","source":"df_train['age'] = (-df_train['mean_birth_259D'] // 365).astype(int)\ndf_train['age']","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:51.863415Z","iopub.execute_input":"2024-10-12T14:36:51.863880Z","iopub.status.idle":"2024-10-12T14:36:51.897397Z","shell.execute_reply.started":"2024-10-12T14:36:51.863828Z","shell.execute_reply":"2024-10-12T14:36:51.895985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. 收入金额是否高于客户平均收入","metadata":{}},{"cell_type":"code","source":"mean = df_train['max_mainoccupationinc_384A'].mean()\ndf_train['income_above_average'] = (df_train['max_mainoccupationinc_384A'] > mean).astype(int)\ndf_train['income_above_average'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:51.909844Z","iopub.execute_input":"2024-10-12T14:36:51.910380Z","iopub.status.idle":"2024-10-12T14:36:51.955212Z","shell.execute_reply.started":"2024-10-12T14:36:51.910335Z","shell.execute_reply":"2024-10-12T14:36:51.953860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. 主要收入金额是否有减少","metadata":{}},{"cell_type":"code","source":"df_train['decrease_in_income'] = (df_train['mean_mainoccupationinc_384A'] - df_train['mean_mainoccupationinc_437A']) < 0\ndf_train['decrease_in_income'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:51.965231Z","iopub.execute_input":"2024-10-12T14:36:51.965856Z","iopub.status.idle":"2024-10-12T14:36:51.994771Z","shell.execute_reply.started":"2024-10-12T14:36:51.965674Z","shell.execute_reply":"2024-10-12T14:36:51.993404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. 贷款占总收入的比例","metadata":{}},{"cell_type":"code","source":"# 避免除以零\ndf_train['CreditToIncomeRatio'] = df_train['totaldebt_9A'].where(df_train['mean_mainoccupationinc_384A'] != 0, df_train['mean_mainoccupationinc_384A']) / df_train['mean_mainoccupationinc_384A']\ndf_train['CreditToIncomeRatio']","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:52.004222Z","iopub.execute_input":"2024-10-12T14:36:52.004820Z","iopub.status.idle":"2024-10-12T14:36:52.031493Z","shell.execute_reply.started":"2024-10-12T14:36:52.004764Z","shell.execute_reply":"2024-10-12T14:36:52.030051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5. 剔除重复内容的部分特征","metadata":{}},{"cell_type":"code","source":"drop=pd.read_excel('/kaggle/input/feature-definitions/drop.xlsx')\nfeatures=[]\nfor i in range(len(drop)):\n    \n    features.append(drop.iloc[i,0])\n\ntrain_df=df_train.drop(columns=features)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:52.035903Z","iopub.execute_input":"2024-10-12T14:36:52.036356Z","iopub.status.idle":"2024-10-12T14:36:53.847185Z","shell.execute_reply.started":"2024-10-12T14:36:52.036313Z","shell.execute_reply":"2024-10-12T14:36:53.845761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 删除重复特征前\nprint(\"train data shape:\\t\", df_train.shape)\nprint(df_train.info())","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:53.849328Z","iopub.execute_input":"2024-10-12T14:36:53.850174Z","iopub.status.idle":"2024-10-12T14:36:53.895616Z","shell.execute_reply.started":"2024-10-12T14:36:53.850113Z","shell.execute_reply":"2024-10-12T14:36:53.894339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 删除重复特征后\n\nprint(\"train data shape:\\t\", train_df.shape)\nprint(train_df.info())","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:36:53.897309Z","iopub.execute_input":"2024-10-12T14:36:53.897743Z","iopub.status.idle":"2024-10-12T14:36:53.931506Z","shell.execute_reply.started":"2024-10-12T14:36:53.897701Z","shell.execute_reply":"2024-10-12T14:36:53.929816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 空值处理","metadata":{}},{"cell_type":"code","source":"drop=pd.read_excel('/kaggle/input/feature-definitions/drop.xlsx')\nfeatures=[]\nfor i in range(len(drop)):\n    features.append(drop.iloc[i,0])\n\ntrain_df=df_train.drop(columns=features)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:37:50.566586Z","iopub.execute_input":"2024-10-12T14:37:50.567779Z","iopub.status.idle":"2024-10-12T14:37:51.888324Z","shell.execute_reply.started":"2024-10-12T14:37:50.567712Z","shell.execute_reply":"2024-10-12T14:37:51.886922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['last_persontype_1072L'].isnull().any()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:37:51.890676Z","iopub.execute_input":"2024-10-12T14:37:51.891148Z","iopub.status.idle":"2024-10-12T14:37:51.906106Z","shell.execute_reply.started":"2024-10-12T14:37:51.891102Z","shell.execute_reply":"2024-10-12T14:37:51.904875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 处理分类特征（有序编码）","metadata":{}},{"cell_type":"code","source":"cat_list = [col for col in train_df.columns if train_df[col].dtype.name == 'category']\n\ncatfreq_dict = {}\ncatcatfreq_dict = {}\n\nfor col in cat_list:\n    catfreq_dict[col] = len(list(train_df[col].value_counts()))\n    catcatfreq_dict[col] = {}\n    for d in dict(train_df[col].value_counts()).items():\n        catcatfreq_dict[col][d[0]] = d[1]\n\ncatfreq_df = pd.DataFrame.from_dict(catfreq_dict, orient='index', columns=['Categories'])\ndisplay(catfreq_df.sort_values(by=\"Categories\", ascending=False).head())","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:37:51.908429Z","iopub.execute_input":"2024-10-12T14:37:51.908847Z","iopub.status.idle":"2024-10-12T14:37:53.156598Z","shell.execute_reply.started":"2024-10-12T14:37:51.908806Z","shell.execute_reply":"2024-10-12T14:37:53.155347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#所有的nan都换成了-1，其余的进行了有序编码（分类变量）\nfrom sklearn.preprocessing import OrdinalEncoder \nordinal_enc = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=np.nan)\ntrain_transformed = ordinal_enc.fit_transform(train_df[cat_list])\ntrain_transformed = np.nan_to_num(train_transformed, nan=-1).astype(int)\ntrain_df[cat_list] = train_transformed","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:37:53.159241Z","iopub.execute_input":"2024-10-12T14:37:53.159646Z","iopub.status.idle":"2024-10-12T14:38:21.538491Z","shell.execute_reply.started":"2024-10-12T14:37:53.159605Z","shell.execute_reply":"2024-10-12T14:38:21.537330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 数值变量（按数据类型和定义选取不同方式进行填充）","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# 创建 nan_list\nnan_list = []\nfor col, boo in train_df.isnull().any().items():\n    if boo == True:\n        nan_list.append(col)\n\n# 将 nan_list 保存到 CSV 文件\npd.Series(nan_list).to_csv('nan_list.csv', index=False)\nprint(f\"Number of col contains Nan value: {len(nan_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:21.540205Z","iopub.execute_input":"2024-10-12T14:38:21.540614Z","iopub.status.idle":"2024-10-12T14:38:22.486441Z","shell.execute_reply.started":"2024-10-12T14:38:21.540572Z","shell.execute_reply":"2024-10-12T14:38:22.485014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:22.487640Z","iopub.execute_input":"2024-10-12T14:38:22.488006Z","iopub.status.idle":"2024-10-12T14:38:24.807949Z","shell.execute_reply.started":"2024-10-12T14:38:22.487967Z","shell.execute_reply":"2024-10-12T14:38:24.806651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:24.809495Z","iopub.execute_input":"2024-10-12T14:38:24.809872Z","iopub.status.idle":"2024-10-12T14:38:24.831412Z","shell.execute_reply.started":"2024-10-12T14:38:24.809834Z","shell.execute_reply":"2024-10-12T14:38:24.830129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int_columns = train_df.select_dtypes(include=['int16', 'int8', 'int32', 'int64'])\nint_columns.isnull().any()\nint_columns.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:24.833360Z","iopub.execute_input":"2024-10-12T14:38:24.833852Z","iopub.status.idle":"2024-10-12T14:38:25.249334Z","shell.execute_reply.started":"2024-10-12T14:38:24.833784Z","shell.execute_reply":"2024-10-12T14:38:25.248131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_columns = train_df.select_dtypes(include=['float16', 'float32'])\nfloat_columns.isnull().any()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:25.254223Z","iopub.execute_input":"2024-10-12T14:38:25.254652Z","iopub.status.idle":"2024-10-12T14:38:26.073415Z","shell.execute_reply.started":"2024-10-12T14:38:25.254610Z","shell.execute_reply":"2024-10-12T14:38:26.071900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#根据下载下来的定义看，只有last_personindex_1023L和last_persontype_1072L是数值行分类变量\n#故对这两列进行用众数填充，其余的用mean填充\nfrom sklearn.impute import SimpleImputer\n# 众数填充数值型分类变量\nfor col in ['last_personindex_1023L', 'last_persontype_1072L']:\n    if train_df[col].isnull().any():\n        mode_value = train_df[col].mode()[0]\n        train_df[col].fillna(mode_value, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:26.074877Z","iopub.execute_input":"2024-10-12T14:38:26.075340Z","iopub.status.idle":"2024-10-12T14:38:26.362426Z","shell.execute_reply.started":"2024-10-12T14:38:26.075297Z","shell.execute_reply":"2024-10-12T14:38:26.361006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Bool数据类型转换\ntrain_df['decrease_in_income'] = train_df['decrease_in_income'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:26.363875Z","iopub.execute_input":"2024-10-12T14:38:26.364314Z","iopub.status.idle":"2024-10-12T14:38:26.378007Z","shell.execute_reply.started":"2024-10-12T14:38:26.364270Z","shell.execute_reply":"2024-10-12T14:38:26.376724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 均值填充普通数值型变量，将原本int数据类型变为float\nnumeric_cols = train_df.select_dtypes(include=['float16', 'float32']).columns\nimp = SimpleImputer(missing_values=np.nan, strategy='mean')\ntrain_df[numeric_cols] = imp.fit_transform(train_df[numeric_cols])","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:26.379574Z","iopub.execute_input":"2024-10-12T14:38:26.380105Z","iopub.status.idle":"2024-10-12T14:38:34.539459Z","shell.execute_reply.started":"2024-10-12T14:38:26.380049Z","shell.execute_reply":"2024-10-12T14:38:34.538296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_list = []\nfor col, boo in train_df.isnull().any().items():\n    if boo == True:\n        nan_list.append(col)\nprint(f\"Number of col contains Nan value: {len(nan_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:34.540841Z","iopub.execute_input":"2024-10-12T14:38:34.541316Z","iopub.status.idle":"2024-10-12T14:38:34.844717Z","shell.execute_reply.started":"2024-10-12T14:38:34.541273Z","shell.execute_reply":"2024-10-12T14:38:34.843266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 检查每个变量是否有缺失值，并打印结果\ni = 0\nj = 0\nfor column in train_df.columns:\n    if train_df[column].isnull().any():\n        i += 1\n    else:\n        j += 1\n\nprint(f\"有缺失值的列个数：{i}\")\nprint(f\"无缺失值的列个数：{j}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:34.846714Z","iopub.execute_input":"2024-10-12T14:38:34.847118Z","iopub.status.idle":"2024-10-12T14:38:35.002689Z","shell.execute_reply.started":"2024-10-12T14:38:34.847077Z","shell.execute_reply":"2024-10-12T14:38:35.001498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:35.004378Z","iopub.execute_input":"2024-10-12T14:38:35.004789Z","iopub.status.idle":"2024-10-12T14:38:35.486619Z","shell.execute_reply.started":"2024-10-12T14:38:35.004747Z","shell.execute_reply":"2024-10-12T14:38:35.485275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:35.488186Z","iopub.execute_input":"2024-10-12T14:38:35.488561Z","iopub.status.idle":"2024-10-12T14:38:35.505091Z","shell.execute_reply.started":"2024-10-12T14:38:35.488523Z","shell.execute_reply":"2024-10-12T14:38:35.503745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop('max_birth_259D', axis=1)\ntrain_df = train_df.drop('dateofbirth_337D', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:35.506874Z","iopub.execute_input":"2024-10-12T14:38:35.507789Z","iopub.status.idle":"2024-10-12T14:38:38.004541Z","shell.execute_reply.started":"2024-10-12T14:38:35.507732Z","shell.execute_reply":"2024-10-12T14:38:38.003147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 相关性分析删除重复变量","metadata":{}},{"cell_type":"code","source":"# 通过相关性分析删去两两间完全相同的float类型变量\ncorrelation_matrix = float_columns.corr()\n\n# 查找相关性等于1 的变量\nhigh_correlation_pairs = []\n\nfor i in range(len(correlation_matrix.columns)):\n    for j in range(i):\n        if abs(correlation_matrix.iloc[i, j]) == 1:\n            high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\n\n# 打印结果\nfor pair in high_correlation_pairs:\n    print(f\"Related variables: {pair[0]} and {pair[1]}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:33:12.913439Z","iopub.execute_input":"2024-10-12T14:33:12.913953Z","iopub.status.idle":"2024-10-12T14:33:12.920488Z","shell.execute_reply.started":"2024-10-12T14:33:12.913902Z","shell.execute_reply":"2024-10-12T14:33:12.919289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop('interestrate_311L', axis=1)\ntrain_df = train_df.drop('max_creationdate_885D', axis=1)\ntrain_df = train_df.drop('max_tenor_203L', axis=1)\ntrain_df = train_df.drop('last_tenor_203L', axis=1)\ntrain_df = train_df.drop('max_mainoccupationinc_384A', axis=1)\ntrain_df = train_df.drop('max_persontype_792L', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:38.014676Z","iopub.execute_input":"2024-10-12T14:38:38.015305Z","iopub.status.idle":"2024-10-12T14:38:43.955337Z","shell.execute_reply.started":"2024-10-12T14:38:38.015258Z","shell.execute_reply":"2024-10-12T14:38:43.953847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop('max_num_group1', axis=1)\ntrain_df = train_df.drop('min_num_group1', axis=1)\ntrain_df = train_df.drop('mean_birth_259D', axis=1)\ntrain_df = train_df.drop('max_num_group1_8', axis=1)\ntrain_df = train_df.drop('min_num_group1_8', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:43.957051Z","iopub.execute_input":"2024-10-12T14:38:43.957441Z","iopub.status.idle":"2024-10-12T14:38:48.429604Z","shell.execute_reply.started":"2024-10-12T14:38:43.957400Z","shell.execute_reply":"2024-10-12T14:38:48.428318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop('case_id', axis=1)\ntrain_df = train_df.drop('WEEK_NUM', axis=1)\ntrain_df = train_df.drop('month_decision', axis=1)\ntrain_df = train_df.drop('weekday_decision', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:48.431310Z","iopub.execute_input":"2024-10-12T14:38:48.431685Z","iopub.status.idle":"2024-10-12T14:38:52.035596Z","shell.execute_reply.started":"2024-10-12T14:38:48.431647Z","shell.execute_reply":"2024-10-12T14:38:52.034387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:38:52.037509Z","iopub.execute_input":"2024-10-12T14:38:52.038010Z","iopub.status.idle":"2024-10-12T14:38:52.062373Z","shell.execute_reply.started":"2024-10-12T14:38:52.037952Z","shell.execute_reply":"2024-10-12T14:38:52.060858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 结合聚类算法的均衡采样方法处理类别不平衡","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.cluster import MiniBatchKMeans\nimport matplotlib.pyplot as plt\n\n# 筛选出target=0的样本\ndf_target_0 = train_df[train_df['target'] == 0].drop('target', axis=1)\n\n# 计算不同数量的聚类中心对应的WCSS（Within-Cluster Sum of Squares）\nwcss = []\nk_values = range(1, 10)  # 测试1到10个聚类中心\nfor k in k_values:\n    mbk = MiniBatchKMeans(n_clusters=k, random_state=42)\n    mbk.fit(df_target_0)\n    wcss.append(mbk.inertia_)\n\n# 绘制碎石图\nplt.figure(figsize=(10, 6))\nplt.plot(k_values, wcss, 'bo-')\nplt.title('Elbow Method')\nplt.xlabel('Number of clusters')\nplt.ylabel('WCSS')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T09:32:34.041073Z","iopub.execute_input":"2024-10-12T09:32:34.041694Z","iopub.status.idle":"2024-10-12T09:33:10.178088Z","shell.execute_reply.started":"2024-10-12T09:32:34.041646Z","shell.execute_reply":"2024-10-12T09:33:10.176880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\n\n\n# 筛选出target=0的样本\ndf_target_0 = train_df[train_df['target'] == 0]\n\n# 标准化特征值\nscaler = StandardScaler()\ndf_target_0_scaled = scaler.fit_transform(df_target_0)\n\n# 使用MiniBatchKMeans进行聚类\noptimal_clusters = 3  \nmbk = MiniBatchKMeans(n_clusters=optimal_clusters, random_state=42, batch_size=1024)\n\n# 训练模型\nmbk.fit(df_target_0_scaled)\n\n# 获取聚类标签\nlabels = mbk.labels_\n\n# 将聚类标签添加到原始数据框中\ndf_target_0['cluster_label'] = labels\n\n# 打印出聚类结果的前几行\nprint(df_target_0.head())","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:40:36.270863Z","iopub.execute_input":"2024-10-12T14:40:36.272286Z","iopub.status.idle":"2024-10-12T14:40:48.701882Z","shell.execute_reply.started":"2024-10-12T14:40:36.272228Z","shell.execute_reply":"2024-10-12T14:40:48.700123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_target_0","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:40:54.926677Z","iopub.execute_input":"2024-10-12T14:40:54.929882Z","iopub.status.idle":"2024-10-12T14:40:56.079713Z","shell.execute_reply.started":"2024-10-12T14:40:54.929778Z","shell.execute_reply":"2024-10-12T14:40:56.077674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# 计算每个cluster_label的样本数量\ncluster_counts = df_target_0['cluster_label'].value_counts()\n\n# 计算总样本数\ntotal_samples = len(df_target_0)\n\n# 计算每个cluster_label的样本比例\ncluster_proportions = cluster_counts / total_samples\n\n# 打印每个cluster_label的样本比例\nprint(cluster_proportions)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:40:56.082667Z","iopub.execute_input":"2024-10-12T14:40:56.083276Z","iopub.status.idle":"2024-10-12T14:40:56.115383Z","shell.execute_reply.started":"2024-10-12T14:40:56.083219Z","shell.execute_reply":"2024-10-12T14:40:56.113781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# 设置随机种子\nnp.random.seed(42)\n\n# 计算抽样比例\ntotal_samples = len(df_target_0)\nsample_size = 246444\nsampling_fraction = sample_size / total_samples\n\n# 进行分层抽样\nsampled_df = df_target_0.groupby('cluster_label', group_keys=False).apply(\n    lambda df: df.sample(frac=sampling_fraction, random_state=42)\n)\n\n# 打印抽样后的DataFrame\nsampled_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:40:56.135293Z","iopub.execute_input":"2024-10-12T14:40:56.136676Z","iopub.status.idle":"2024-10-12T14:40:59.933595Z","shell.execute_reply.started":"2024-10-12T14:40:56.136598Z","shell.execute_reply":"2024-10-12T14:40:59.932095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# 计算每个cluster_label的样本数量\ncluster_counts = sampled_df['cluster_label'].value_counts()\n\n# 计算总样本数\ntotal_samples = len(sampled_df)\n\n# 计算每个cluster_label的样本比例\ncluster_proportions = cluster_counts / total_samples\n\n# 打印每个cluster_label的样本比例\nprint(cluster_proportions)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:40:59.936268Z","iopub.execute_input":"2024-10-12T14:40:59.936709Z","iopub.status.idle":"2024-10-12T14:40:59.948490Z","shell.execute_reply.started":"2024-10-12T14:40:59.936662Z","shell.execute_reply":"2024-10-12T14:40:59.947152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# 筛选出train_df中target=1的样本，并去除target列\ndf_target_1 = train_df[train_df['target'] == 1]\n\n# 合并sampled_df和df_target_1\ncombined_df = pd.concat([sampled_df, df_target_1])","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:41:00.032320Z","iopub.execute_input":"2024-10-12T14:41:00.033554Z","iopub.status.idle":"2024-10-12T14:41:00.506571Z","shell.execute_reply.started":"2024-10-12T14:41:00.033496Z","shell.execute_reply":"2024-10-12T14:41:00.505184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:41:00.508940Z","iopub.execute_input":"2024-10-12T14:41:00.509431Z","iopub.status.idle":"2024-10-12T14:41:00.628003Z","shell.execute_reply.started":"2024-10-12T14:41:00.509387Z","shell.execute_reply":"2024-10-12T14:41:00.626730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_df = combined_df.drop('cluster_label', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:41:00.657172Z","iopub.execute_input":"2024-10-12T14:41:00.657653Z","iopub.status.idle":"2024-10-12T14:41:00.826376Z","shell.execute_reply.started":"2024-10-12T14:41:00.657611Z","shell.execute_reply":"2024-10-12T14:41:00.825081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom imblearn.over_sampling import SMOTE\n\n# 目标变量是'target'\nX = combined_df.drop('target', axis=1)  # 特征矩阵\ny = combined_df['target']  # 目标变量\n\n# 创建SMOTE对象\nsmote = SMOTE(random_state=42)\n\n# 执行过采样\nX_resampled, y_resampled = smote.fit_resample(X, y)\n\n# 将过采样后的数据转换回DataFrame\nresampled_df = pd.DataFrame(X_resampled, columns=X.columns)\nresampled_df['target'] = y_resampled\n\n# 打印过采样后的数据集形状，以确认类别平衡\nprint(resampled_df['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:41:01.027845Z","iopub.execute_input":"2024-10-12T14:41:01.028332Z","iopub.status.idle":"2024-10-12T14:41:44.070154Z","shell.execute_reply.started":"2024-10-12T14:41:01.028288Z","shell.execute_reply":"2024-10-12T14:41:44.068628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resampled_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:42:07.050927Z","iopub.execute_input":"2024-10-12T14:42:07.052108Z","iopub.status.idle":"2024-10-12T14:42:07.393584Z","shell.execute_reply.started":"2024-10-12T14:42:07.052045Z","shell.execute_reply":"2024-10-12T14:42:07.392091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resampled_df.to_parquet('train_bal.parquet', index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T09:59:43.128796Z","iopub.execute_input":"2024-10-12T09:59:43.129628Z","iopub.status.idle":"2024-10-12T09:59:49.201480Z","shell.execute_reply.started":"2024-10-12T09:59:43.129540Z","shell.execute_reply":"2024-10-12T09:59:49.200086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 分箱与WOE编码","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.tree import DecisionTreeClassifier\nfrom category_encoders.woe import WOEEncoder\n\n\ndef decision_tree_binning(df, feature, target, max_bins):\n    # 使用决策树分类器进行分箱\n    model = DecisionTreeClassifier(max_leaf_nodes=max_bins)\n    model.fit(df[[feature]], df[target])\n    \n    # 获取每个样本的叶子节点\n    bins = model.apply(df[[feature]])\n    df[f'{feature}_binned'] = bins\n\n    # 确保每个箱中都有好样本和坏样本\n    def has_both_classes(group):\n        return group[target].nunique() > 1\n\n    grouped = df.groupby(f'{feature}_binned').filter(has_both_classes)\n\n    # 如果有任何箱不符合条件，调整分箱\n    while len(grouped) < len(df) and max_bins > 2:\n        max_bins -= 1\n        model = DecisionTreeClassifier(max_leaf_nodes=max_bins)\n        model.fit(df[[feature]], df[target])\n        bins = model.apply(df[[feature]])\n        df[f'{feature}_binned'] = bins\n        grouped = df.groupby(f'{feature}_binned').filter(has_both_classes)\n\n    return df[f'{feature}_binned']\n\n# 示例用法\n# df = pd.DataFrame({'feature': [...], 'target': [...]})\n# binned_feature = decision_tree_binning(df, 'feature', 'target', max_bins=5)\n\n\n# 示例用法\n# df = pd.DataFrame({'feature': [...], 'target': [...]})\n# binned_feature = decision_tree_binning(df, 'feature', 'target', max_bins=5)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:42:24.350440Z","iopub.execute_input":"2024-10-12T14:42:24.350898Z","iopub.status.idle":"2024-10-12T14:42:24.845786Z","shell.execute_reply.started":"2024-10-12T14:42:24.350856Z","shell.execute_reply":"2024-10-12T14:42:24.844442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# WOE 编码函数\ndef woe_iv_encoding(df, feature, target):\n    encoder = WOEEncoder(cols=[f'{feature}_binned'])\n    df_transformed = encoder.fit_transform(df[[f'{feature}_binned']], df[[target]])\n    \n    grouped = df.groupby(f'{feature}_binned')\n    total_good = df[target].value_counts(normalize=True)[0] * len(df)\n    total_bad = df[target].value_counts(normalize=True)[1] * len(df)\n    \n    woe_list = []\n    iv_list = []\n    \n    for name, group in grouped:\n        value_counts = group[target].value_counts(normalize=True)\n        G_i = value_counts.get(0, 0)\n        B_i = value_counts.get(1, 0)\n\n        if B_i == 0 or G_i == 0:\n            continue\n\n        woe_i = np.log((G_i + 1e-6) / (B_i + 1e-6))\n        woe_list.append(woe_i)\n\n        iv_i = (G_i - B_i) * woe_i\n        iv_list.append(iv_i)\n\n    total_iv = np.sum(iv_list)\n    woe_encoded = df_transformed[f'{feature}_binned']\n\n    return woe_encoded, total_iv, woe_list, iv_list\n\n# 对每个特征进行分箱和 WOE 编码的主函数\ndef encode_features(df, excluded_columns, max_bins=10):\n    iv_values = {}\n    woe_columns = []\n    bins_info = {}\n    \n    # 先保存原始列名\n    original_columns = df.columns.difference(excluded_columns)\n    \n    for feature in original_columns:\n        if df[feature].dtype in ['object', 'category']:\n            label_encoder = LabelEncoder()\n            df[feature] = label_encoder.fit_transform(df[feature])\n            \n        bins = decision_tree_binning(df, feature, 'target', max_bins)\n        bins_info[feature] = len(np.unique(bins))\n        \n        woe_encoded, iv, woe_list, iv_list = woe_iv_encoding(df, feature, 'target')\n        iv_values[feature] = iv\n        woe_columns.append(f'{feature}_woe')\n        df[f'{feature}_woe'] = woe_encoded\n\n\n    return df, iv_values, woe_columns, bins_info","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:42:27.747761Z","iopub.execute_input":"2024-10-12T14:42:27.748543Z","iopub.status.idle":"2024-10-12T14:42:27.766134Z","shell.execute_reply.started":"2024-10-12T14:42:27.748485Z","shell.execute_reply":"2024-10-12T14:42:27.764834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 调用主函数\nwoe_df, iv_values, woe_columns, bins_info = encode_features(resampled_df, excluded_columns=['target'])","metadata":{"execution":{"iopub.status.busy":"2024-10-12T14:42:32.629504Z","iopub.execute_input":"2024-10-12T14:42:32.630104Z","iopub.status.idle":"2024-10-12T15:20:21.588552Z","shell.execute_reply.started":"2024-10-12T14:42:32.630043Z","shell.execute_reply":"2024-10-12T15:20:21.587113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:22:40.262081Z","iopub.execute_input":"2024-10-12T15:22:40.262957Z","iopub.status.idle":"2024-10-12T15:22:40.465850Z","shell.execute_reply.started":"2024-10-12T15:22:40.262904Z","shell.execute_reply":"2024-10-12T15:22:40.464204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 创建一个新的DataFrame来存储WOE编码后的列\nwoe_df_transformed = woe_df[woe_columns]\n\n# 将 iv_values 转换为 DataFrame\niv_df = pd.DataFrame(list(iv_values.items()), columns=['Feature', 'IV'])\n\nwoediff_num = []\n\nfor columns in woe_df_transformed:\n    # 获取列的不同值\n    unique_values = woe_df_transformed[columns].unique()\n#     print(unique_values)\n    num_unique_values = len(unique_values)\n#     print(f'共有 {num_unique_values} 种不同的值。')\n    woediff_num.append(num_unique_values)\n    \nprint(woediff_num)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:22:42.922287Z","iopub.execute_input":"2024-10-12T15:22:42.922746Z","iopub.status.idle":"2024-10-12T15:22:44.394330Z","shell.execute_reply.started":"2024-10-12T15:22:42.922704Z","shell.execute_reply":"2024-10-12T15:22:44.392980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_num = []\nfor feature, num_bins in bins_info.items():\n#     print(f\"{feature} is binned into {num_bins} bins\")\n    bin_num.append(num_bins)\n    \nprint(bin_num)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:22:49.052461Z","iopub.execute_input":"2024-10-12T15:22:49.052945Z","iopub.status.idle":"2024-10-12T15:22:49.060636Z","shell.execute_reply.started":"2024-10-12T15:22:49.052895Z","shell.execute_reply":"2024-10-12T15:22:49.059271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_df_transformed","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:22:51.685003Z","iopub.execute_input":"2024-10-12T15:22:51.685623Z","iopub.status.idle":"2024-10-12T15:22:51.950753Z","shell.execute_reply.started":"2024-10-12T15:22:51.685554Z","shell.execute_reply":"2024-10-12T15:22:51.949506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 检查列表长度是否与DataFrame的行数相同\nif len(woediff_num) == len(iv_df):\n    # 将列表添加为新列\n    iv_df['woediff_num'] = woediff_num\nelse:\n    print(\"列表长度与DataFrame的行数不匹配\")\n    \n\nif len(bin_num) == len(iv_df):\n    # 将列表添加为新列\n    iv_df['bin_num'] = bin_num\nelse:\n    print(\"列表长度与DataFrame的行数不匹配\")\n\n# iv_df = iv_df.drop('diff_num', axis=1)\n\n# 打印结果查看\niv_df","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:26.428012Z","iopub.execute_input":"2024-10-12T15:23:26.428482Z","iopub.status.idle":"2024-10-12T15:23:26.450461Z","shell.execute_reply.started":"2024-10-12T15:23:26.428442Z","shell.execute_reply":"2024-10-12T15:23:26.449227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iv_df.info()\niv_df.to_csv('iv_df_adj_1.csv', index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:32.238452Z","iopub.execute_input":"2024-10-12T15:23:32.238921Z","iopub.status.idle":"2024-10-12T15:23:32.258315Z","shell.execute_reply.started":"2024-10-12T15:23:32.238877Z","shell.execute_reply":"2024-10-12T15:23:32.256530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_df_transformed['target'] = woe_df['target']","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:36.612498Z","iopub.execute_input":"2024-10-12T15:23:36.613003Z","iopub.status.idle":"2024-10-12T15:23:36.621417Z","shell.execute_reply.started":"2024-10-12T15:23:36.612956Z","shell.execute_reply":"2024-10-12T15:23:36.620175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:39.294530Z","iopub.execute_input":"2024-10-12T15:23:39.295092Z","iopub.status.idle":"2024-10-12T15:23:39.502373Z","shell.execute_reply.started":"2024-10-12T15:23:39.295013Z","shell.execute_reply":"2024-10-12T15:23:39.500971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_df_transformed.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:44.003586Z","iopub.execute_input":"2024-10-12T15:23:44.004088Z","iopub.status.idle":"2024-10-12T15:23:44.020982Z","shell.execute_reply.started":"2024-10-12T15:23:44.004010Z","shell.execute_reply":"2024-10-12T15:23:44.019334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 通过IV进行特征筛选","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# 绘制 'IV' 列的直方图\niv_df['IV'].hist()\nplt.title('Distribution of IV')\nplt.xlabel('IV')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:46.370407Z","iopub.execute_input":"2024-10-12T15:23:46.370904Z","iopub.status.idle":"2024-10-12T15:23:46.772587Z","shell.execute_reply.started":"2024-10-12T15:23:46.370856Z","shell.execute_reply":"2024-10-12T15:23:46.771125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_iv_df = iv_df\n# 删除 'IV' 列值大于0.8\nnew_iv_df = new_iv_df[(new_iv_df['IV'] < 3)  & (new_iv_df['IV'] > 0.02)]\n\n# 创建一个新的索引列\nnew_iv_df['new_index'] = range(1, len(new_iv_df) + 1)\n\n#打印结果查看\nprint(new_iv_df)\nnew_iv_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:50.132189Z","iopub.execute_input":"2024-10-12T15:23:50.132652Z","iopub.status.idle":"2024-10-12T15:23:50.153439Z","shell.execute_reply.started":"2024-10-12T15:23:50.132611Z","shell.execute_reply":"2024-10-12T15:23:50.152013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_df_adj = woe_df_transformed","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:56.156720Z","iopub.execute_input":"2024-10-12T15:23:56.157375Z","iopub.status.idle":"2024-10-12T15:23:56.163007Z","shell.execute_reply.started":"2024-10-12T15:23:56.157316Z","shell.execute_reply":"2024-10-12T15:23:56.161824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取 new_iv_df['Feature'] 中的所有列名\ncolumns_to_keep = new_iv_df['Feature'].apply(lambda x: x + '_woe')\n\n# 从 woe_df_adj 中保留这些列\nwoe_df_adj = woe_df_adj.loc[:, woe_df_adj.columns.isin(columns_to_keep)]\nwoe_df_adj['target'] = woe_df_transformed['target']\n\n# 打印结果查看\nwoe_df_adj","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:23:59.207565Z","iopub.execute_input":"2024-10-12T15:23:59.208140Z","iopub.status.idle":"2024-10-12T15:23:59.654595Z","shell.execute_reply.started":"2024-10-12T15:23:59.208094Z","shell.execute_reply":"2024-10-12T15:23:59.653107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# WOE编码后（未删除IV大于等于0.8的特征）\nwoe_df_transformed.to_parquet('woe_df_1.parquet', index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T13:34:55.310858Z","iopub.execute_input":"2024-10-12T13:34:55.311284Z","iopub.status.idle":"2024-10-12T13:34:58.338871Z","shell.execute_reply.started":"2024-10-12T13:34:55.311247Z","shell.execute_reply":"2024-10-12T13:34:58.337664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# WOE编码后（删除IV大于等于0.8的特征）\nwoe_df_adj.to_parquet('woe_df_2.parquet', index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T13:35:01.883514Z","iopub.execute_input":"2024-10-12T13:35:01.883995Z","iopub.status.idle":"2024-10-12T13:35:04.152802Z","shell.execute_reply.started":"2024-10-12T13:35:01.883933Z","shell.execute_reply":"2024-10-12T13:35:04.151399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_iv_df.to_csv('iv_df_adj_2.csv', index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-12T13:35:10.654770Z","iopub.execute_input":"2024-10-12T13:35:10.655207Z","iopub.status.idle":"2024-10-12T13:35:10.663468Z","shell.execute_reply.started":"2024-10-12T13:35:10.655166Z","shell.execute_reply":"2024-10-12T13:35:10.662156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 打印每个变量的分箱结果好样本和坏样本的比例\ndef print_bin_rates(df, feature, target):\n    # 计算每个箱子中好样本和坏样本的数量\n    bin_counts = df.groupby(f'{feature}_binned')[target].value_counts(normalize=True)\n    # 将坏样本标记为1，好样本标记为0\n    bin_rates = bin_counts.unstack(fill_value=0)\n    # 计算坏样本比例\n    bad_rate = bin_rates[1]\n    # 计算好样本比例\n    good_rate = bin_rates[0]\n    # 打印结果\n    print(f\"Bin rates for feature '{feature}':\")\n    print(\"Bad rate (1) per bin:\")\n    print(bad_rate)\n    print(\"\\nGood rate (0) per bin:\")\n    print(good_rate)\n\n\nexcluded_columns=['target']\n\n# 遍历每个变量并打印分箱结果\nfor feature in woe_df.columns.difference(excluded_columns):\n    if f'{feature}_binned' in woe_df.columns:\n        print_bin_rates(woe_df, feature, 'target')","metadata":{"execution":{"iopub.status.busy":"2024-10-12T15:24:26.036959Z","iopub.execute_input":"2024-10-12T15:24:26.037472Z","iopub.status.idle":"2024-10-12T15:24:31.697631Z","shell.execute_reply.started":"2024-10-12T15:24:26.037428Z","shell.execute_reply":"2024-10-12T15:24:31.695940Z"},"trusted":true},"execution_count":null,"outputs":[]}]}