{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":9509911,"sourceType":"datasetVersion","datasetId":5738294}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nimport joblib\nimport lightgbm as lgb\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:24.850593Z","iopub.execute_input":"2024-09-29T17:20:24.851021Z","iopub.status.idle":"2024-09-29T17:20:29.058182Z","shell.execute_reply.started":"2024-09-29T17:20:24.850969Z","shell.execute_reply":"2024-09-29T17:20:29.056898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#数据所在地址\nROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:35.490403Z","iopub.execute_input":"2024-09-29T17:20:35.491019Z","iopub.status.idle":"2024-09-29T17:20:35.496620Z","shell.execute_reply.started":"2024-09-29T17:20:35.490975Z","shell.execute_reply":"2024-09-29T17:20:35.495347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_base = read_file(TRAIN_DIR / \"train_base.parquet\")\n# df_base.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T13:10:16.577460Z","iopub.execute_input":"2024-09-29T13:10:16.577897Z","iopub.status.idle":"2024-09-29T13:10:16.742112Z","shell.execute_reply.started":"2024-09-29T13:10:16.577855Z","shell.execute_reply":"2024-09-29T13:10:16.740936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#读取定义文件\nfeature_definitions = pd.read_csv(\"/kaggle/input/feature-definitions/feature_definitions.csv\")\nfeature_definitions","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:38.525098Z","iopub.execute_input":"2024-09-29T17:20:38.526228Z","iopub.status.idle":"2024-09-29T17:20:38.565677Z","shell.execute_reply.started":"2024-09-29T17:20:38.526177Z","shell.execute_reply":"2024-09-29T17:20:38.564516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#读取更新后定义文件\ndata_train_df_feature_all = pd.read_csv(\"/kaggle/input/feature-definitions/data_train_df_feature_all.csv\")\ndata_train_df_feature_all","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:40.935420Z","iopub.execute_input":"2024-09-29T17:20:40.935880Z","iopub.status.idle":"2024-09-29T17:20:40.957769Z","shell.execute_reply.started":"2024-09-29T17:20:40.935832Z","shell.execute_reply":"2024-09-29T17:20:40.956645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df = data_train_df_feature_all[data_train_df_feature_all['Column Name'].str.endswith('D')]\nfiltered_df","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:43.314986Z","iopub.execute_input":"2024-09-29T17:20:43.315511Z","iopub.status.idle":"2024-09-29T17:20:43.335007Z","shell.execute_reply.started":"2024-09-29T17:20:43.315460Z","shell.execute_reply":"2024-09-29T17:20:43.333472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = feature_definitions[feature_definitions['Variable'] == 'date_decision']\nrow","metadata":{"execution":{"iopub.status.busy":"2024-09-29T13:40:02.991655Z","iopub.execute_input":"2024-09-29T13:40:02.992989Z","iopub.status.idle":"2024-09-29T13:40:03.006446Z","shell.execute_reply.started":"2024-09-29T13:40:02.992929Z","shell.execute_reply":"2024-09-29T13:40:03.005124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 对数据做类型转换和筛选","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n    #设置表格数据类型\n    @staticmethod\n    def set_table_dtypes(df): #Standardize the dtype.\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    #处理日期，将D的特性更改为date_decision的天数差。\n    @staticmethod\n    def handle_dates(df): \n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    #过滤列\n    #如果列名不在保留列表中，并且列的空值比例大于0.95，则删除该列\n    #如果列名不在保留列表中，并且列的数据类型是String，且唯一值数量为1或大于200，则删除该列\n    @staticmethod\n#     def filter_cols(df): \n#         for col in df.columns:\n#             if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n#                 isnull = df[col].is_null().mean()\n\n#                 if isnull > 0.95:\n#                     df = df.drop(col)\n\n#         for col in df.columns:\n#             if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n#                 freq = df[col].n_unique()\n#                 if (freq == 1) | (freq > 200):\n#                     df = df.drop(col)\n        \n#         return df\n\n\n\n#过滤的方法2\n    @staticmethod\n    def filter_cols(df): \n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                if col[-1] in (\"P\",\"A\") :\n                    #连续的空值大于0.3删除\n                    isnull = df[col].is_null().mean()\n                    if isnull > 0.30:\n                        df = df.drop(col)\n                else:\n                     #其他类型空值大于0.5删除\n                    isnull = df[col].is_null().mean()\n                    if isnull > 0.50:\n                        df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                #后续特征封箱freq大于200可不用删除\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:47.100504Z","iopub.execute_input":"2024-09-29T17:20:47.101476Z","iopub.status.idle":"2024-09-29T17:20:47.116172Z","shell.execute_reply.started":"2024-09-29T17:20:47.101430Z","shell.execute_reply":"2024-09-29T17:20:47.114575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 自动聚合\n定义了一个名为Aggregator的类，其中包含多个静态方法，用于生成聚合表达式","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n    \n    #为数值类型的列生成聚合表达式\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        \n#         expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        return expr_max +expr_mean\n    \n    #为日期类型的列生成聚合表达式\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n#         expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        return  expr_max +expr_mean\n    \n    #为字符串类型的列生成聚合表达式\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        #expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n        return  expr_max +expr_last#+expr_count\n    \n    #为未指定类型的列生成聚合表达式\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return  expr_max +expr_last\n    \n    #提取每个num_group的最大值和最小值\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] \n        expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n#         expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return  expr_max +expr_min\n    \n    #执行上面的函数并返回结果\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:49.025297Z","iopub.execute_input":"2024-09-29T17:20:49.026155Z","iopub.status.idle":"2024-09-29T17:20:49.039558Z","shell.execute_reply.started":"2024-09-29T17:20:49.026103Z","shell.execute_reply":"2024-09-29T17:20:49.038358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Aggregator:\n#     #为数值类型的列生成最大值，最小值聚合表达式\n#     @staticmethod\n#     def num_expr(df): \n#         cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n#         expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n#         expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n\n#         return expr_max, expr_min\n    \n#     #为日期类型的列生成最大值聚合表达式\n#     @staticmethod\n#     def date_expr(df):\n#         cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n#         expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n#         expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n\n#         return expr_max, expr_min\n    \n#     #为字符串类型的列生成最大值聚合表达式\n#     @staticmethod\n#     def str_expr(df): \n#         cols = [col for col in df.columns if col[-1] in (\"M\",)]\n# #         expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n#         expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        \n#         # 统计每个字符串列中出现次数最多的值\n#         for col in cols:\n#             # 计算每个值的出现次数\n#             count_expr = df.group_by(pl.col(col)).count()\n#             print(count_expr)\n#             # 找到出现次数最多的值\n#             max_count_expr = count_expr.max().alias(f\"max_count_{col}\")\n#             print(max_count_expr)\n#             # 创建一个新的聚合表达式，用于获取每个值的出现次数\n# #             count_agg_expr = pl.col(f\"count_{col}\").alias(f\"count_{col}\")\n#             # 将新的聚合表达式添加到列表中\n# #             expr_max.append(count_agg_expr)\n            \n#         return max_count_expr,expr_min\n# #         return expr_max, expr_min\n    \n#     #为未指定类型的列生成最大值聚合表达式\n#     @staticmethod\n#     def other_expr(df): \n#         cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        \n#         expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n#         print(expr_max)\n#         expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n\n#         return expr_max, expr_min\n    \n    \n#     #提取每个num_group的最大值和最小值\n#     @staticmethod\n#     def count_expr(df): \n#         cols = [col for col in df.columns if \"num_group\" in col]\n#         expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n#         expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n\n#         return expr_max, expr_min\n\n#     #执行上面的函数并返回结果\n#     @staticmethod\n#     def get_exprs(df): \n#         maxexprs = Aggregator.num_expr(df)[0] + \\\n#                 Aggregator.date_expr(df)[0] + \\\n#                 Aggregator.str_expr(df)[0] + \\\n#                 Aggregator.other_expr(df)[0] + \\\n#                 Aggregator.count_expr(df)[0]\n        \n#         minexprs = Aggregator.num_expr(df)[1] + \\\n#                 Aggregator.date_expr(df)[1] + \\\n#                 Aggregator.str_expr(df)[1] + \\\n#                 Aggregator.other_expr(df)[1] + \\\n#                 Aggregator.count_expr(df)[1]\n\n#         return maxexprs, minexprs","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:46:10.679308Z","iopub.execute_input":"2024-09-22T02:46:10.679821Z","iopub.status.idle":"2024-09-22T02:46:10.697464Z","shell.execute_reply.started":"2024-09-22T02:46:10.679768Z","shell.execute_reply":"2024-09-22T02:46:10.695948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取文件的函数","metadata":{}},{"cell_type":"code","source":"#对于单个文件读取\ndef read_file(path, depth=None): \n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n     #·分组聚合·->·多条case·id的记录 按照 最大值 进行合并\n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg( Aggregator.get_exprs(df))\n    \n    return df\n\n#对于多个文件读取\ndef read_files(regex_path, depth=None):\n    chunks = []\n    #glob用于通配符，读取之后最后的_0_1不一样的那些文件\n    for path in glob(str(regex_path)):\n        chunks.append(pl.read_parquet(path).pipe(Pipeline.set_table_dtypes))\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\") #拼接的操作concat，按行\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg( Aggregator.get_exprs(df))\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:52.250928Z","iopub.execute_input":"2024-09-29T17:20:52.251383Z","iopub.status.idle":"2024-09-29T17:20:52.259807Z","shell.execute_reply.started":"2024-09-29T17:20:52.251343Z","shell.execute_reply":"2024-09-29T17:20:52.258342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 特征工程函数，用于添加新特征和合并数据框","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    \n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:54.340794Z","iopub.execute_input":"2024-09-29T17:20:54.341219Z","iopub.status.idle":"2024-09-29T17:20:54.348204Z","shell.execute_reply.started":"2024-09-29T17:20:54.341180Z","shell.execute_reply":"2024-09-29T17:20:54.346936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#转换成dataframe的数据格式\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:56.045500Z","iopub.execute_input":"2024-09-29T17:20:56.046641Z","iopub.status.idle":"2024-09-29T17:20:56.056140Z","shell.execute_reply.started":"2024-09-29T17:20:56.046578Z","shell.execute_reply":"2024-09-29T17:20:56.054693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 读取训练集数据和特征工程","metadata":{}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:20:57.889681Z","iopub.execute_input":"2024-09-29T17:20:57.890076Z","iopub.status.idle":"2024-09-29T17:21:26.216280Z","shell.execute_reply.started":"2024-09-29T17:20:57.890033Z","shell.execute_reply":"2024-09-29T17:21:26.214897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:21:39.140010Z","iopub.execute_input":"2024-09-29T17:21:39.140466Z","iopub.status.idle":"2024-09-29T17:21:50.182595Z","shell.execute_reply.started":"2024-09-29T17:21:39.140425Z","shell.execute_reply":"2024-09-29T17:21:50.181277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#筛选数据（字符串类别和大量空值数据）\ndf_train = df_train.pipe(Pipeline.filter_cols)\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:21:51.480490Z","iopub.execute_input":"2024-09-29T17:21:51.481004Z","iopub.status.idle":"2024-09-29T17:21:54.743524Z","shell.execute_reply.started":"2024-09-29T17:21:51.480957Z","shell.execute_reply":"2024-09-29T17:21:54.742223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:22:01.254763Z","iopub.execute_input":"2024-09-29T17:22:01.255304Z","iopub.status.idle":"2024-09-29T17:22:01.277849Z","shell.execute_reply.started":"2024-09-29T17:22:01.255235Z","shell.execute_reply":"2024-09-29T17:22:01.276350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#减少内存函数\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:22:04.550809Z","iopub.execute_input":"2024-09-29T17:22:04.551214Z","iopub.status.idle":"2024-09-29T17:22:04.567330Z","shell.execute_reply.started":"2024-09-29T17:22:04.551177Z","shell.execute_reply":"2024-09-29T17:22:04.565800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()\ndf_train, cat_cols = to_pandas(df_train)\ndf_train = reduce_mem_usage(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:22:09.795068Z","iopub.execute_input":"2024-09-29T17:22:09.795483Z","iopub.status.idle":"2024-09-29T17:22:28.411377Z","shell.execute_reply.started":"2024-09-29T17:22:09.795446Z","shell.execute_reply":"2024-09-29T17:22:28.410029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 后续的一些处理——增加和删除变量","metadata":{}},{"cell_type":"markdown","source":"1. 申请时的客户年龄（后续分析年龄分布与逾期情况）","metadata":{}},{"cell_type":"code","source":"df_train['age']=-df_train['mean_birth_259D']//365\ndf_train['age']","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:02.716113Z","iopub.execute_input":"2024-09-29T17:25:02.716700Z","iopub.status.idle":"2024-09-29T17:25:02.733439Z","shell.execute_reply.started":"2024-09-29T17:25:02.716646Z","shell.execute_reply":"2024-09-29T17:25:02.731982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. 收入金额是否高于客户平均收入","metadata":{}},{"cell_type":"code","source":"mean=df_train['max_mainoccupationinc_384A'].mean()\ndf_train['income_above_average']=df_train['max_mainoccupationinc_384A'].apply(lambda x: 1 if x>mean else 0)\ndf_train['income_above_average'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:05.654799Z","iopub.execute_input":"2024-09-29T17:25:05.655303Z","iopub.status.idle":"2024-09-29T17:25:08.929015Z","shell.execute_reply.started":"2024-09-29T17:25:05.655242Z","shell.execute_reply":"2024-09-29T17:25:08.927865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. 主要收入金额是否有减少","metadata":{}},{"cell_type":"code","source":"df_train['decrease_in_income']=(df_train['mean_mainoccupationinc_384A']-df_train['mean_mainoccupationinc_437A']).apply(lambda x: 1 if x<0 else 0)\ndf_train['decrease_in_income'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:11.654931Z","iopub.execute_input":"2024-09-29T17:25:11.655770Z","iopub.status.idle":"2024-09-29T17:25:12.393589Z","shell.execute_reply.started":"2024-09-29T17:25:11.655718Z","shell.execute_reply":"2024-09-29T17:25:12.392315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. 贷款占总收入的比例","metadata":{}},{"cell_type":"code","source":"df_train['CreditToIncomeRatio'] = df_train['totaldebt_9A'] / df_train['mean_mainoccupationinc_384A']\ndf_train['CreditToIncomeRatio']","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:15.325332Z","iopub.execute_input":"2024-09-29T17:25:15.325758Z","iopub.status.idle":"2024-09-29T17:25:15.343074Z","shell.execute_reply.started":"2024-09-29T17:25:15.325717Z","shell.execute_reply":"2024-09-29T17:25:15.341739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5. 剔除重复内容的部分特征","metadata":{}},{"cell_type":"code","source":"drop=pd.read_excel('/kaggle/input/feature-definitions/drop.xlsx')\nfeatures=[]\nfor i in range(len(drop)):\n    features.append(drop.iloc[i,0])\n\ntrain_df=df_train.drop(columns=features)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:20.285412Z","iopub.execute_input":"2024-09-29T17:25:20.285843Z","iopub.status.idle":"2024-09-29T17:25:21.849094Z","shell.execute_reply.started":"2024-09-29T17:25:20.285795Z","shell.execute_reply":"2024-09-29T17:25:21.847607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:40:28.795681Z","iopub.execute_input":"2024-09-29T17:40:28.796160Z","iopub.status.idle":"2024-09-29T17:40:28.805871Z","shell.execute_reply.started":"2024-09-29T17:40:28.796109Z","shell.execute_reply":"2024-09-29T17:40:28.804545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train data shape:\\t\", train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:31.385188Z","iopub.execute_input":"2024-09-29T17:25:31.386051Z","iopub.status.idle":"2024-09-29T17:25:31.392130Z","shell.execute_reply.started":"2024-09-29T17:25:31.386004Z","shell.execute_reply":"2024-09-29T17:25:31.390840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#没有做空值的处理，原始数据\ntrain_df.to_parquet('train_df（initial）.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:25:47.630476Z","iopub.execute_input":"2024-09-29T17:25:47.631417Z","iopub.status.idle":"2024-09-29T17:26:03.578485Z","shell.execute_reply.started":"2024-09-29T17:25:47.631365Z","shell.execute_reply":"2024-09-29T17:26:03.576940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 对于空值后续处理1(简单处理)","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:28:42.187116Z","iopub.execute_input":"2024-09-29T17:28:42.188754Z","iopub.status.idle":"2024-09-29T17:28:42.231703Z","shell.execute_reply.started":"2024-09-29T17:28:42.188693Z","shell.execute_reply":"2024-09-29T17:28:42.230474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 处理分类特征（序数编码）","metadata":{}},{"cell_type":"code","source":"cat_list = [col for col in train_df.columns if train_df[col].dtype.name == 'category']\n\ncatfreq_dict = {}\ncatcatfreq_dict = {}\n\nfor col in cat_list:\n    catfreq_dict[col] = len(list(train_df[col].value_counts()))\n    catcatfreq_dict[col] = {}\n    for d in dict(train_df[col].value_counts()).items():\n        catcatfreq_dict[col][d[0]] = d[1]\n\ncatfreq_df = pd.DataFrame.from_dict(catfreq_dict, orient='index', columns=['Categories'])\ndisplay(catfreq_df.sort_values(by=\"Categories\", ascending=False).head())","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:29:43.446245Z","iopub.execute_input":"2024-09-29T17:29:43.447504Z","iopub.status.idle":"2024-09-29T17:29:44.750366Z","shell.execute_reply.started":"2024-09-29T17:29:43.447453Z","shell.execute_reply":"2024-09-29T17:29:44.748978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#所有的nan都换成了-1，其余的进行了有序编码（分类变量）\nfrom sklearn.preprocessing import OrdinalEncoder \nordinal_enc = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=np.nan)\ntrain_transformed = ordinal_enc.fit_transform(train_df[cat_list])\ntrain_transformed = np.nan_to_num(train_transformed, nan=-1).astype(int)\ntrain_df[cat_list] = train_transformed","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:30:58.495714Z","iopub.execute_input":"2024-09-29T17:30:58.496165Z","iopub.status.idle":"2024-09-29T17:31:23.007889Z","shell.execute_reply.started":"2024-09-29T17:30:58.496126Z","shell.execute_reply":"2024-09-29T17:31:23.006707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[cat_list]","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:31:47.981402Z","iopub.execute_input":"2024-09-29T17:31:47.981863Z","iopub.status.idle":"2024-09-29T17:31:49.365493Z","shell.execute_reply.started":"2024-09-29T17:31:47.981821Z","shell.execute_reply":"2024-09-29T17:31:49.364095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2024-09-29T16:52:09.646277Z","iopub.execute_input":"2024-09-29T16:52:09.646819Z","iopub.status.idle":"2024-09-29T16:52:10.279597Z","shell.execute_reply.started":"2024-09-29T16:52:09.646771Z","shell.execute_reply":"2024-09-29T16:52:10.278253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 数值变量","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:35:32.785431Z","iopub.execute_input":"2024-09-29T17:35:32.785894Z","iopub.status.idle":"2024-09-29T17:35:32.808873Z","shell.execute_reply.started":"2024-09-29T17:35:32.785856Z","shell.execute_reply":"2024-09-29T17:35:32.807324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_list = []\nfor col, boo in train_df.isnull().any().items():\n    if boo == True:\n        nan_list.append(col)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:39:18.481505Z","iopub.execute_input":"2024-09-29T17:39:18.482435Z","iopub.status.idle":"2024-09-29T17:39:19.480807Z","shell.execute_reply.started":"2024-09-29T17:39:18.482370Z","shell.execute_reply":"2024-09-29T17:39:19.479648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:40:41.740348Z","iopub.execute_input":"2024-09-29T17:40:41.741179Z","iopub.status.idle":"2024-09-29T17:40:44.397312Z","shell.execute_reply.started":"2024-09-29T17:40:41.741132Z","shell.execute_reply":"2024-09-29T17:40:44.395977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nimp = SimpleImputer(missing_values=np.nan, strategy='mean')\ntrain_df[nan_list] = imp.fit_transform(train_df[nan_list])","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:40:48.380948Z","iopub.execute_input":"2024-09-29T17:40:48.381459Z","iopub.status.idle":"2024-09-29T17:40:54.621697Z","shell.execute_reply.started":"2024-09-29T17:40:48.381413Z","shell.execute_reply":"2024-09-29T17:40:54.620360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:40:57.011885Z","iopub.execute_input":"2024-09-29T17:40:57.012346Z","iopub.status.idle":"2024-09-29T17:40:57.231768Z","shell.execute_reply.started":"2024-09-29T17:40:57.012301Z","shell.execute_reply":"2024-09-29T17:40:57.230658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:41:00.275551Z","iopub.execute_input":"2024-09-29T17:41:00.275984Z","iopub.status.idle":"2024-09-29T17:41:00.283625Z","shell.execute_reply.started":"2024-09-29T17:41:00.275945Z","shell.execute_reply":"2024-09-29T17:41:00.282359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#简单处理（分类变量做有序编码，数值变量做mean填充）\ntrain_df.to_parquet('train_df_1（simple）.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:41:01.535173Z","iopub.execute_input":"2024-09-29T17:41:01.535627Z","iopub.status.idle":"2024-09-29T17:41:13.371610Z","shell.execute_reply.started":"2024-09-29T17:41:01.535586Z","shell.execute_reply":"2024-09-29T17:41:13.370223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 空值处理2（更精细一些的）","metadata":{}},{"cell_type":"code","source":"drop=pd.read_excel('/kaggle/input/feature-definitions/drop.xlsx')\nfeatures=[]\nfor i in range(len(drop)):\n    features.append(drop.iloc[i,0])\n\ntrain_df=df_train.drop(columns=features)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:41:41.125438Z","iopub.execute_input":"2024-09-29T17:41:41.125896Z","iopub.status.idle":"2024-09-29T17:41:42.140341Z","shell.execute_reply.started":"2024-09-29T17:41:41.125852Z","shell.execute_reply":"2024-09-29T17:41:42.138899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['last_persontype_1072L'].isnull().any()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:53:49.701572Z","iopub.execute_input":"2024-09-29T17:53:49.702048Z","iopub.status.idle":"2024-09-29T17:53:49.715753Z","shell.execute_reply.started":"2024-09-29T17:53:49.702005Z","shell.execute_reply":"2024-09-29T17:53:49.714516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 处理分类特征（有序编码）","metadata":{}},{"cell_type":"code","source":"cat_list = [col for col in train_df.columns if train_df[col].dtype.name == 'category']\n\ncatfreq_dict = {}\ncatcatfreq_dict = {}\n\nfor col in cat_list:\n    catfreq_dict[col] = len(list(train_df[col].value_counts()))\n    catcatfreq_dict[col] = {}\n    for d in dict(train_df[col].value_counts()).items():\n        catcatfreq_dict[col][d[0]] = d[1]\n\ncatfreq_df = pd.DataFrame.from_dict(catfreq_dict, orient='index', columns=['Categories'])\ndisplay(catfreq_df.sort_values(by=\"Categories\", ascending=False).head())","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:42:03.321108Z","iopub.execute_input":"2024-09-29T17:42:03.321633Z","iopub.status.idle":"2024-09-29T17:42:04.531993Z","shell.execute_reply.started":"2024-09-29T17:42:03.321588Z","shell.execute_reply":"2024-09-29T17:42:04.530693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#所有的nan都换成了-1，其余的进行了有序编码（分类变量）\nfrom sklearn.preprocessing import OrdinalEncoder \nordinal_enc = OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=np.nan)\ntrain_transformed = ordinal_enc.fit_transform(train_df[cat_list])\ntrain_transformed = np.nan_to_num(train_transformed, nan=-1).astype(int)\ntrain_df[cat_list] = train_transformed","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:42:07.910368Z","iopub.execute_input":"2024-09-29T17:42:07.911713Z","iopub.status.idle":"2024-09-29T17:42:32.747733Z","shell.execute_reply.started":"2024-09-29T17:42:07.911649Z","shell.execute_reply":"2024-09-29T17:42:32.746542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 数值变量（按数据类型和定义选取不同方式进行填充）","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n# 创建 nan_list\nnan_list = []\nfor col, boo in train_df.isnull().any().items():\n    if boo == True:\n        nan_list.append(col)\n\n# 将 nan_list 保存到 CSV 文件\npd.Series(nan_list).to_csv('nan_list.csv', index=False)\nprint(f\"Number of col contains Nan value: {len(nan_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:45:05.810835Z","iopub.execute_input":"2024-09-29T17:45:05.812216Z","iopub.status.idle":"2024-09-29T17:45:06.758784Z","shell.execute_reply.started":"2024-09-29T17:45:05.812162Z","shell.execute_reply":"2024-09-29T17:45:06.757530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:57:07.087152Z","iopub.execute_input":"2024-09-29T17:57:07.087732Z","iopub.status.idle":"2024-09-29T17:57:09.533371Z","shell.execute_reply.started":"2024-09-29T17:57:07.087676Z","shell.execute_reply":"2024-09-29T17:57:09.532168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#根据下载下来的定义看，只有last_personindex_1023L和last_persontype_1072L是数值行分类变量\n#故对这两列进行用众数填充，其余的用mean填充\nfrom sklearn.impute import SimpleImputer\n# 众数填充数值型分类变量\nfor col in ['last_personindex_1023L', 'last_persontype_1072L']:\n    if train_df[col].isnull().any():\n        mode_value = train_df[col].mode()[0]\n        train_df[col].fillna(mode_value, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T17:58:37.446468Z","iopub.execute_input":"2024-09-29T17:58:37.446928Z","iopub.status.idle":"2024-09-29T17:58:37.548828Z","shell.execute_reply.started":"2024-09-29T17:58:37.446887Z","shell.execute_reply":"2024-09-29T17:58:37.547416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 均值填充普通数值型变量\nnumeric_cols = train_df.select_dtypes(include=['number']).columns.difference(['last_personindex_1023L', 'last_persontype_1072L'])\nimp = SimpleImputer(missing_values=np.nan, strategy='mean')\ntrain_df[numeric_cols] = imp.fit_transform(train_df[numeric_cols])","metadata":{"execution":{"iopub.status.busy":"2024-09-29T18:03:00.540996Z","iopub.execute_input":"2024-09-29T18:03:00.541484Z","iopub.status.idle":"2024-09-29T18:03:13.864842Z","shell.execute_reply.started":"2024-09-29T18:03:00.541438Z","shell.execute_reply":"2024-09-29T18:03:13.863627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-29T18:05:03.720578Z","iopub.execute_input":"2024-09-29T18:05:03.721034Z","iopub.status.idle":"2024-09-29T18:05:03.729018Z","shell.execute_reply.started":"2024-09-29T18:05:03.720986Z","shell.execute_reply":"2024-09-29T18:05:03.727730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_list = []\nfor col, boo in train_df.isnull().any().items():\n    if boo == True:\n        nan_list.append(col)\nprint(f\"Number of col contains Nan value: {len(nan_list)}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-29T18:05:47.970773Z","iopub.execute_input":"2024-09-29T18:05:47.971285Z","iopub.status.idle":"2024-09-29T18:05:48.472385Z","shell.execute_reply.started":"2024-09-29T18:05:47.971220Z","shell.execute_reply":"2024-09-29T18:05:48.470933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#更精细处理（分类变量做有序编码，数值分类做众数，普通数值做mean）\ntrain_df.to_parquet('train_df_2（complex）.parquet', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-29T18:05:56.876147Z","iopub.execute_input":"2024-09-29T18:05:56.877471Z","iopub.status.idle":"2024-09-29T18:06:08.546289Z","shell.execute_reply.started":"2024-09-29T18:05:56.877400Z","shell.execute_reply":"2024-09-29T18:06:08.545049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 其他","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# 假设column_name是你想要查看分布的列名\ntrain_df['decrease_in_income'].hist(bins=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T15:18:47.595784Z","iopub.execute_input":"2024-09-29T15:18:47.596301Z","iopub.status.idle":"2024-09-29T15:18:47.983465Z","shell.execute_reply.started":"2024-09-29T15:18:47.596253Z","shell.execute_reply":"2024-09-29T15:18:47.982122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nums=df_train.select_dtypes(exclude='category').columns\nnums_df = df_train[nums]\n# nums_df.dytype","metadata":{"execution":{"iopub.status.busy":"2024-09-29T14:53:38.154895Z","iopub.execute_input":"2024-09-29T14:53:38.155927Z","iopub.status.idle":"2024-09-29T14:53:40.908658Z","shell.execute_reply.started":"2024-09-29T14:53:38.155880Z","shell.execute_reply":"2024-09-29T14:53:40.907330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 得到筛选后的特征\n此时数据为220个特征\n下面的data_train_all为筛选变量之后的全部数据(1526659, 220)  \ndata_train_feature_all为剩余这220个变量的的特征，可以对应特征定义的表格进行找相关联的关系\n对应在右边的output中可以下载  \n这两个文件里面既有数值变量也有字符串变量等","metadata":{}},{"cell_type":"code","source":"# df_train.write_csv('/kaggle/working/data_train_all.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:46:53.768449Z","iopub.execute_input":"2024-09-22T02:46:53.768837Z","iopub.status.idle":"2024-09-22T02:47:03.031816Z","shell.execute_reply.started":"2024-09-22T02:46:53.768797Z","shell.execute_reply":"2024-09-22T02:47:03.030642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 获取列名\n# columns = df_train.columns\n# df_feature = pd.DataFrame(columns, columns=['Column Name'])\n# df_feature\n# df_feature.to_csv('/kaggle/working/data_train_feature_all.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:54:41.367467Z","iopub.execute_input":"2024-09-22T02:54:41.367882Z","iopub.status.idle":"2024-09-22T02:54:41.380622Z","shell.execute_reply.started":"2024-09-22T02:54:41.367843Z","shell.execute_reply":"2024-09-22T02:54:41.379487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 筛选那些数值型变量，排除分类变量\n此时数据为160个特征,都是数值型的\n下面的data_train_num为筛选出数值型变量之后的全部数据(1526659, 160)  \ndata_train_feature_num为剩余这160个变量的的特征，可以对应特征定义的表格进行找相关联的关系\n对应在右边的output中可以下载  \n这两个文件里面只有数值变量","metadata":{}},{"cell_type":"code","source":"# df_train, cat_cols = to_pandas(df_train)\n# data_train_num = df_train.select_dtypes(exclude='category')\n# data_train_num.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:50:02.752218Z","iopub.execute_input":"2024-09-22T02:50:02.752731Z","iopub.status.idle":"2024-09-22T02:50:18.469222Z","shell.execute_reply.started":"2024-09-22T02:50:02.752686Z","shell.execute_reply":"2024-09-22T02:50:18.467676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_train_num.to_csv('/kaggle/working/data_train_num.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:47:03.445243Z","iopub.status.idle":"2024-09-22T02:47:03.445699Z","shell.execute_reply.started":"2024-09-22T02:47:03.445468Z","shell.execute_reply":"2024-09-22T02:47:03.445489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # 获取列名\n# columns_num = data_train_num.columns\n# df_feature_num = pd.DataFrame(columns_num, columns=['Column Name'])\n# df_feature_num\n# df_feature_num.to_csv('/kaggle/working/data_train_feature_num.csv')\n# # df_feature_num","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:47:03.447472Z","iopub.status.idle":"2024-09-22T02:47:03.447966Z","shell.execute_reply.started":"2024-09-22T02:47:03.447760Z","shell.execute_reply":"2024-09-22T02:47:03.447784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # 获取列名\n# columns = df_train.columns\n# df_feature = pd.DataFrame(columns, columns=['Column Name'])\n# # df_feature\n# df_feature.to_csv('/kaggle/working/data_train_feature_all.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-22T02:47:03.449449Z","iopub.status.idle":"2024-09-22T02:47:03.449928Z","shell.execute_reply.started":"2024-09-22T02:47:03.449728Z","shell.execute_reply":"2024-09-22T02:47:03.449750Z"},"trusted":true},"execution_count":null,"outputs":[]}]}