{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"},{"sourceId":150384981,"sourceType":"kernelVersion"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Created by yunsuxiaozi\n### Install lightautoml","metadata":{}},{"cell_type":"code","source":"#https://www.kaggle.com/code/cody11null/lgbm-x2-nn#LightAutoML-NN-(DenseLight)-prediction\n#安装包,--no-index 表示不从 PyPI（Python Package Index）上下载安装包\n#-U是upgrade -q是减少输出信息,--find-links=是指定链接 后面是安装的版本.\n!pip install --no-index -U -q --find-links=/kaggle/input/lightautoml-038-dependecies lightautoml==0.3.8\n!pip install --no-index -U -q --find-links=/kaggle/input/lightautoml-038-dependecies pandas==2.0.3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import necessary libraries","metadata":{}},{"cell_type":"code","source":"#necessary\nimport polars as pl#和pandas类似,但是处理大型数据集有更好的性能.\nimport pandas as pd#导入csv文件的库\nimport numpy as np#进行矩阵运算的库\n\n#model\n# LightAutoML presets, task and report generation\nfrom lightautoml.automl.presets.tabular_presets import TabularAutoML#自动化表格机器学习模型的库\nfrom lightautoml.tasks import Task#定义机器学习任务的库\n#metric\nfrom sklearn.metrics import roc_auc_score#导入roc_auc曲线\nimport dill#对对象进行序列化和反序列化(例如保存和加载树模型)\n#KFold是直接分成k折,StratifiedKFold还要考虑每种类别的占比\nfrom sklearn.model_selection import StratifiedKFold\nimport gc#垃圾回收模块\nfrom sklearn.decomposition import TruncatedSVD#截断奇异值分解,是一种数据降维的方法\n\nimport time#标准库的时间模块\n#为了方便后期调用训练的模型时不会调用错版本,提供模型训练的时间\n#time.strftime()函数用于将时间对象格式化为字符串，time.localtime()函数返回表示当前本地时间的time.struct_time对象\ncurrent_time = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\nprint(\"this notebook training time is \", current_time)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Config and set random seed","metadata":{}},{"cell_type":"code","source":"#config\nclass Config():\n    seed=2024\n    num_folds=10\n    TARGET_NAME ='target'\n    batch_size=1000#由于不知道测试数据的大小,所以分批次放入模型.\nimport random#提供了一些用于生成随机数的函数\n#设置随机种子,保证模型可以复现\ndef seed_everything(seed):\n    np.random.seed(seed)#numpy的随机种子\n    random.seed(seed)#python内置的随机种子\nseed_everything(Config.seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### data preprocessor","metadata":{}},{"cell_type":"code","source":"#这里是抄 https://www.kaggle.com/code/andreynesterov/home-credit-baseline-data\n#有些需要concat的列列名相同,dtype不同,需要做统一,暂时不对时间列做处理\ndef set_table_dtypes(df):\n    for col in df.columns:\n        if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n            df = df.with_columns(pl.col(col).cast(pl.Int64).alias(col))\n#         elif col==\"date_decision\":\n#             df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        elif col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        elif col[-1] in (\"M\"):\n            df = df.with_columns(pl.col(col).cast(pl.String).alias(col))\n#         elif col[-1] in (\"D\",):\n#             df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))            \n    return df\n\n\n\ndef preprocessor(mode='train'):#mode='train'|'test'\n    #base 文件\n    print(\"base file\")\n    feats=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_base.csv\")\n    #求一下星期几\n    feats=feats.with_columns(pl.col(\"date_decision\").cast(pl.Date).dt.weekday().alias(\"weekday\"))  \n        \n    print(\"applprev_1 file num 2(3)\")#这里保留最新的数据\n    applprev1_0=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_applprev_1_0.csv\")\n    applprev1_1=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_applprev_1_1.csv\")\n    applprev1=pl.concat([applprev1_0,applprev1_1],how=\"vertical_relaxed\")\n    if mode=='test':\n        applprev1_2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_applprev_1_2.csv\")\n        applprev1=pl.concat([applprev1,applprev1_2],how=\"vertical_relaxed\")\n        \n    applprev1=applprev1.with_columns(pl.col('case_id').shift(-1).alias(\"case_id_shift_1\"))\n    #如果case_id不等于下一个case_id,说明它是最后一个数据了,保留下来\n    applprev1=applprev1.filter(pl.col('case_id')!=pl.col(\"case_id_shift_1\"))\n    applprev1=applprev1.drop([\"case_id_shift_1\"])\n    feats=feats.join(applprev1,on='case_id',how='left')\n    del applprev1,applprev1_0,applprev1_1\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n        \n        \n    print(\"applprev_2 file num 1\")#这里保留最新的数据\n    applprev=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_applprev_2.csv\")\n    applprev=applprev.with_columns(pl.col('case_id').shift(-1).alias(\"case_id_shift_1\"))\n    #如果case_id不等于下一个case_id,说明它是最后一个数据了,保留下来\n    applprev=applprev.filter(pl.col('case_id')!=pl.col(\"case_id_shift_1\"))\n    applprev=applprev.drop([\"case_id_shift_1\"])\n    feats=feats.join(applprev,on='case_id',how='left')\n    del applprev\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n    \n    print(\"credit_bureau file num 2\")\n    credit_bureau_1=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_credit_bureau_b_1.csv\")\n    credit_bureau_2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_credit_bureau_b_2.csv\")\n    \n    credit_bureau=credit_bureau_1.join(credit_bureau_2,on='case_id',how='left')\n    credit_bureau = credit_bureau.fill_null(-1)\n    cols=['amount_1115A', 'credlmt_1052A', 'credlmt_228A', 'credlmt_3940954A', 'credquantity_1099L', 'credquantity_984L', 'debtpastduevalue_732A', 'debtvalue_227A', 'dpd_550P', 'dpd_733P', 'dpdmax_851P', 'dpdmaxdatemonth_804T', 'dpdmaxdateyear_742T', 'installmentamount_644A', 'installmentamount_833A', 'instlamount_892A', 'maxdebtpduevalodued_3940955A', 'num_group1', 'numberofinstls_810L', 'overdueamountmax_950A', 'overdueamountmaxdatemonth_494T', 'overdueamountmaxdateyear_432T', 'pmtdaysoverdue_1135P', 'pmtnumpending_403L', 'residualamount_1093A', 'residualamount_127A', 'residualamount_3940956A', 'totalamount_503A', 'totalamount_881A', 'num_group1_right', 'num_group2', 'pmts_dpdvalue_108P', 'pmts_pmtsoverdue_635A']\n    #数值列的特征工程  从1开始是为了把'case_id'去掉    \n    for col in cols:\n        column_type = credit_bureau[col].dtype\n        is_numeric = (column_type == pl.datatypes.Int64) or (column_type == pl.datatypes.Float64) \n        if is_numeric:#数值列构造特征\n            feat=credit_bureau.group_by('case_id').agg( pl.max(col).alias(f\"max_credit_bureau_{col}\"),\n                                           pl.mean(col).alias(f\"mean_credit_bureau_{col}\"),\n                                           pl.median(col).alias(f\"median_credit_bureau_{col}\"),\n                                           pl.std(col).alias(f\"std_credit_bureau_{col}\"),\n                                           pl.min(col).alias(f\"min_credit_bureau_{col}\"),\n                                           pl.count(col).alias(f\"count_credit_bureau_{col}\"),\n                                           pl.sum(col).alias(f\"sum_credit_bureau_{col}\"),\n                                           pl.n_unique(col).alias(f\"n_unique_credit_bureau_{col}\"),\n                                           pl.first(col).alias(f\"first_credit_bureau_{col}\"),\n                                           pl.last(col).alias(f\"last_credit_bureau_{col}\")\n                                         )\n            feats=feats.join(feat,on='case_id',how='left')\n    \n    del credit_bureau_1,credit_bureau_2,credit_bureau\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n    \n    print(\"debitcard file num 1\")\n    debitcard=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_debitcard_1.csv\")\n    ##数值列的特征工程  从1开始是为了把'case_id'去掉    \n    for idx in range(1,len(debitcard.columns)):\n        col=debitcard.columns[idx]\n        column_type = debitcard[col].dtype\n        is_numeric = (column_type == pl.datatypes.Int64) or (column_type == pl.datatypes.Float64) \n        if is_numeric:#数值列构造特征\n            feat=debitcard.group_by('case_id').agg( pl.max(col).alias(f\"max_debitcard_{col}\"),\n                                           pl.mean(col).alias(f\"mean_debitcard_{col}\"),\n                                           pl.median(col).alias(f\"median_debitcard_{col}\"),\n                                           pl.std(col).alias(f\"std_debitcard_{col}\"),\n                                           pl.min(col).alias(f\"min_debitcard_{col}\"),\n                                           pl.count(col).alias(f\"count_debitcard_{col}\"),\n                                           pl.sum(col).alias(f\"sum_debitcard_{col}\"),\n                                           pl.n_unique(col).alias(f\"n_unique_debitcard_{col}\"),\n                                           pl.first(col).alias(f\"first_debitcard_{col}\"),\n                                           pl.last(col).alias(f\"last_debitcard_{col}\")\n                                         )\n            feats=feats.join(feat,on='case_id',how='left')\n    \n    del debitcard\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n    \n    \n    print(\"deposit file num 1\")\n    deposit=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_deposit_1.csv\")\n    #数值列的特征工程  从1开始是为了把'case_id'去掉    \n    for idx in range(1,len(deposit.columns)):\n        col=deposit.columns[idx]\n        column_type = deposit[col].dtype\n        is_numeric = (column_type == pl.datatypes.Int64) or (column_type == pl.datatypes.Float64) \n        if is_numeric:#数值列构造特征\n            feat=deposit.group_by('case_id').agg( pl.max(col).alias(f\"max_deposit_{col}\"),\n                                           pl.mean(col).alias(f\"mean_deposit_{col}\"),\n                                           pl.median(col).alias(f\"median_deposit_{col}\"),\n                                           pl.std(col).alias(f\"std_deposit_{col}\"),\n                                           pl.min(col).alias(f\"min_deposit_{col}\"),\n                                           pl.count(col).alias(f\"count_deposit_{col}\"),\n                                           pl.sum(col).alias(f\"sum_deposit_{col}\"),\n                                           pl.n_unique(col).alias(f\"n_unique_deposit_{col}\"),\n                                           pl.first(col).alias(f\"first_deposit_{col}\"),\n                                           pl.last(col).alias(f\"last_deposit_{col}\")\n                                         )\n            feats=feats.join(feat,on='case_id',how='left')\n\n    #经过检查person2训练集和测试集对应的列dtype都对应的上\n    person2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_person_2.csv\")\n    #数值列的特征工程  从1开始是为了把'case_id'去掉    \n    for idx in range(1,len(person2.columns)):\n        col=person2.columns[idx]\n        column_type= person2[col].dtype\n        is_numeric = (column_type == pl.datatypes.Int64) or (column_type == pl.datatypes.Float64) \n        if is_numeric:#数值列构造特征\n            feat=person2.group_by('case_id').agg( pl.max(col).alias(f\"max_person2_{col}\"),\n                                           pl.mean(col).alias(f\"mean_person2_{col}\"),\n                                           pl.median(col).alias(f\"median_person2_{col}\"),\n                                           pl.std(col).alias(f\"std_person2_{col}\"),\n                                           pl.min(col).alias(f\"min_person2_{col}\"),\n                                           pl.count(col).alias(f\"count_person2_{col}\"),\n                                           pl.sum(col).alias(f\"sum_person2_{col}\"),\n                                           pl.n_unique(col).alias(f\"n_unique_person2_{col}\"),\n                                           pl.first(col).alias(f\"first_person2_{col}\"),\n                                           pl.last(col).alias(f\"last_person2_{col}\")\n                                         )\n            feats=feats.join(feat,on='case_id',how='left')\n    del person2\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n      \n    #static_0,训练数据2个文件,测试数据3个文件（关于这里的处理,我自己处理总是会threw exception,所以抄https://www.kaggle.com/code/jetakow/home-credit-2024-starter-notebook）\n    print(f\"static_0 file num 2(3)\")\n    #pipe用于在DataFrame上自定义自己的函数\n    static_0_0=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_0_0.csv\").pipe(set_table_dtypes)\n    static_0_1=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_0_1.csv\").pipe(set_table_dtypes)\n    \n    static=pl.concat([static_0_0,static_0_1],how=\"vertical_relaxed\")#垂直合并,并且放宽了数据类型匹配的限制\n    if mode=='test':#如果是测试数据的话还有一个文件\n        static_0_2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_0_2.csv\").pipe(set_table_dtypes)\n        static=pl.concat([static,static_0_2],how=\"vertical_relaxed\")\n    feats=feats.join(static,on='case_id',how='left')\n    del static,static_0_0,static_0_1\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n    \n    #static_cb文件\n    print(\"static_cb_file num 1\")\n    static_cb=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_cb_0.csv\")\n    feats=feats.join(static_cb,on='case_id',how='left')\n    del static_cb\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n     \n    #tax的3个文件(tax_c这个文件好像存在些问题,暂时不放入训练数据)\n    print(\"tax_a,b file num 2\")\n    tax_a=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_tax_registry_a_1.csv\")\n    tax_b=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_tax_registry_b_1.csv\")\n    tax=tax_a.join(tax_b,on=['case_id','num_group1'],how='left')\n    #数值列的特征工程  从1开始是为了把'case_id'去掉    \n    for idx in range(1,len(tax.columns)):\n        col=tax.columns[idx]\n        column_type = tax[col].dtype\n        is_numeric = (column_type == pl.datatypes.Int64) or (column_type == pl.datatypes.Float64) \n        if is_numeric:#数值列构造特征\n            feat=tax.group_by('case_id').agg( pl.max(col).alias(f\"max_tax_{col}\"),\n                                           pl.mean(col).alias(f\"mean_tax_{col}\"),\n                                           pl.median(col).alias(f\"median_tax_{col}\"),\n                                           pl.std(col).alias(f\"std_tax_{col}\"),\n                                           pl.min(col).alias(f\"min_tax_{col}\"),\n                                           pl.count(col).alias(f\"count_tax_{col}\"),\n                                           pl.sum(col).alias(f\"sum_tax_{col}\"),\n                                           pl.n_unique(col).alias(f\"n_unique_tax_{col}\"),\n                                           pl.first(col).alias(f\"first_tax_{col}\"),\n                                           pl.last(col).alias(f\"last_tax_{col}\")\n                                         )\n            feats=feats.join(feat,on='case_id',how='left')\n    del tax_a,tax_b,tax\n    gc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存\n    \n    #对month的处理\n    feats=feats.with_columns((pl.col('MONTH')%100).alias(\"MONTH\"))\n    \n    return feats\n\nprint(\"---------Feature Engineer-------------\\n\")\ntrain_feats=preprocessor(mode='train')\n#由于这是时间序列数据,加上数据量太大,构造太多的特征就会超内存了,所以这里考虑就用最后一年的数据来训练模型,以达到最好效果.\n#这里的时间差不多是从2019年10月初开始.\ntrain_feats=train_feats.filter(train_feats['WEEK_NUM']>=40)\nprint(f\"len(train_feats):{len(train_feats)}\")\n\ntest_feats=preprocessor(mode='test')\nprint(f\"len(test_feats):{len(test_feats)}\")\n\nprint(\"---------Feature Transformer-------------\\n\")\n\"\"\"\n下面是对特征进行处理,提取出来的特征最初是有字符串类型和数值类型.\n1.先对字符串 类别型变量 进行 one hot encoder\n2.剩下的字符串直接去掉,一列有唯一值的去掉,缺失值占比>0.95的列选择drop.\n这样数据中就只有数值列了.\n3.填充缺失值,这里考虑用-1填充.\n4.数值列先将相关性高达0.99的列只保留一列,其余去掉.\n5.对相关性高的那些列进行降维处理.\n\"\"\"\n\n#如果打开的两个文件的相同列一个是浮点数类型,一个是object或者str,就把两个都转成浮点数类型.\nfor col in test_feats.columns:\n    if (train_feats[col].dtype==pl.datatypes.Float64) or (test_feats[col].dtype==pl.datatypes.Float64):\n        train_feats.with_columns(train_feats[col].cast(pl.datatypes.Float64))\n        test_feats.with_columns(test_feats[col].cast(pl.datatypes.Float64))\ntrain_feats=train_feats.to_pandas()\ntest_feats=test_feats.to_pandas()\n\n# #检查有没有训练数据某列和测试数据某列dtype不同的情况.\n# for col in test_feats.columns:\n#     if test_feats[col].dtype!=train_feats[col].dtype:\n#         print(\"error!!!!!!!!!!\")\n\n#对字符串特征列进行独热编码的转换\nprint(\"----------string one hot encoder ****\")\nfor col in test_feats.columns:\n    n_unique=train_feats[col].nunique()\n    #如果是类别型变量的话,独热编码转换\n    #如果类别是2类,像性别一样,如果是(0,1)了,或者说数值类型的话,没必要转换.如果是字符串类型的话,转换成数值\n    if n_unique==2 and train_feats[col].dtype=='object':\n        print(f\"one_hot_2:{col}\")\n        unique=train_feats[col].unique()\n        #随便选择一个类别进行转换,比如gender='Female'\n        train_feats[col]=(train_feats[col]==unique[0]).astype(int)\n        test_feats[col]=(test_feats[col]==unique[0]).astype(int)\n    elif (n_unique<10) and train_feats[col].dtype=='object':#由于内存有限 类别型变量的n_unique设置为20\n        print(f\"one_hot_10:{col}\")\n        unique=train_feats[col].unique()\n        for idx in range(len(unique)):\n            if unique[idx]==unique[idx]:#这里是为了避免字符串中存在nan值的情况\n                train_feats[col+\"_\"+str(idx)]=(train_feats[col]==unique[idx]).astype(int)\n                test_feats[col+\"_\"+str(idx)]=(test_feats[col]==unique[idx]).astype(int)\n        train_feats.drop([col],axis=1,inplace=True)\n        test_feats.drop([col],axis=1,inplace=True)\n\n#如果是字符串的列或者一列只有唯一值,去掉\nprint(\"----------drop other string or unique value full null value ****\")\n# drop_cols=[]\n# for col in test_feats.columns:\n#     if (train_feats[col].dtype=='object') or (test_feats[col].dtype=='object') \\\n#         or (train_feats[col].nunique()==1) or train_feats[col].isna().mean()>0.95:\n#         drop_cols+=[col]\n# #case_id目前看来和id一样没什么用,WEEK_NUM在测试数据中比训练数据大.\n# drop_cols+=['case_id','WEEK_NUM']\n\ndrop_cols=['date_decision', 'approvaldate_319D', 'byoccupationinc_3656910L', 'cancelreason_3545846M', 'childnum_21L', 'creationdate_885D', 'credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_transactions_402L', 'currdebt_94A', 'dateactivated_425D', 'district_544M', 'dtlastpmt_581D', 'dtlastpmtallstes_3545839D', 'employedfrom_700D', 'firstnonzeroinstldate_307D', 'isdebitcard_527L', 'maxdpdtolerance_577P', 'outstandingdebt_522A', 'profession_152M', 'rejectreason_755M', 'rejectreasonclient_4145042M', 'revolvingaccount_394A', 'status_219L', 'max_credit_bureau_amount_1115A', 'mean_credit_bureau_amount_1115A', 'median_credit_bureau_amount_1115A', 'std_credit_bureau_amount_1115A', 'min_credit_bureau_amount_1115A', 'count_credit_bureau_amount_1115A', 'sum_credit_bureau_amount_1115A', 'n_unique_credit_bureau_amount_1115A', 'first_credit_bureau_amount_1115A', 'last_credit_bureau_amount_1115A', 'max_credit_bureau_credlmt_1052A', 'mean_credit_bureau_credlmt_1052A', 'median_credit_bureau_credlmt_1052A', 'std_credit_bureau_credlmt_1052A', 'min_credit_bureau_credlmt_1052A', 'count_credit_bureau_credlmt_1052A', 'sum_credit_bureau_credlmt_1052A', 'n_unique_credit_bureau_credlmt_1052A', 'first_credit_bureau_credlmt_1052A', 'last_credit_bureau_credlmt_1052A', 'max_credit_bureau_credlmt_228A', 'mean_credit_bureau_credlmt_228A', 'median_credit_bureau_credlmt_228A', 'std_credit_bureau_credlmt_228A', 'min_credit_bureau_credlmt_228A', 'count_credit_bureau_credlmt_228A', 'sum_credit_bureau_credlmt_228A', 'n_unique_credit_bureau_credlmt_228A', 'first_credit_bureau_credlmt_228A', 'last_credit_bureau_credlmt_228A', 'max_credit_bureau_credlmt_3940954A', 'mean_credit_bureau_credlmt_3940954A', 'median_credit_bureau_credlmt_3940954A', 'std_credit_bureau_credlmt_3940954A', 'min_credit_bureau_credlmt_3940954A', 'count_credit_bureau_credlmt_3940954A', 'sum_credit_bureau_credlmt_3940954A', 'n_unique_credit_bureau_credlmt_3940954A', 'first_credit_bureau_credlmt_3940954A', 'last_credit_bureau_credlmt_3940954A', 'max_credit_bureau_credquantity_1099L', 'mean_credit_bureau_credquantity_1099L', 'median_credit_bureau_credquantity_1099L', 'std_credit_bureau_credquantity_1099L', 'min_credit_bureau_credquantity_1099L', 'count_credit_bureau_credquantity_1099L', 'sum_credit_bureau_credquantity_1099L', 'n_unique_credit_bureau_credquantity_1099L', 'first_credit_bureau_credquantity_1099L', 'last_credit_bureau_credquantity_1099L', 'max_credit_bureau_credquantity_984L', 'mean_credit_bureau_credquantity_984L', 'median_credit_bureau_credquantity_984L', 'std_credit_bureau_credquantity_984L', 'min_credit_bureau_credquantity_984L', 'count_credit_bureau_credquantity_984L', 'sum_credit_bureau_credquantity_984L', 'n_unique_credit_bureau_credquantity_984L', 'first_credit_bureau_credquantity_984L', 'last_credit_bureau_credquantity_984L', 'max_credit_bureau_debtpastduevalue_732A', 'mean_credit_bureau_debtpastduevalue_732A', 'median_credit_bureau_debtpastduevalue_732A', 'std_credit_bureau_debtpastduevalue_732A', 'min_credit_bureau_debtpastduevalue_732A', 'count_credit_bureau_debtpastduevalue_732A', 'sum_credit_bureau_debtpastduevalue_732A', 'n_unique_credit_bureau_debtpastduevalue_732A', 'first_credit_bureau_debtpastduevalue_732A', 'last_credit_bureau_debtpastduevalue_732A', 'max_credit_bureau_debtvalue_227A', 'mean_credit_bureau_debtvalue_227A', 'median_credit_bureau_debtvalue_227A', 'std_credit_bureau_debtvalue_227A', 'min_credit_bureau_debtvalue_227A', 'count_credit_bureau_debtvalue_227A', 'sum_credit_bureau_debtvalue_227A', 'n_unique_credit_bureau_debtvalue_227A', 'first_credit_bureau_debtvalue_227A', 'last_credit_bureau_debtvalue_227A', 'max_credit_bureau_dpd_550P', 'mean_credit_bureau_dpd_550P', 'median_credit_bureau_dpd_550P', 'std_credit_bureau_dpd_550P', 'min_credit_bureau_dpd_550P', 'count_credit_bureau_dpd_550P', 'sum_credit_bureau_dpd_550P', 'n_unique_credit_bureau_dpd_550P', 'first_credit_bureau_dpd_550P', 'last_credit_bureau_dpd_550P', 'max_credit_bureau_dpd_733P', 'mean_credit_bureau_dpd_733P', 'median_credit_bureau_dpd_733P', 'std_credit_bureau_dpd_733P', 'min_credit_bureau_dpd_733P', 'count_credit_bureau_dpd_733P', 'sum_credit_bureau_dpd_733P', 'n_unique_credit_bureau_dpd_733P', 'first_credit_bureau_dpd_733P', 'last_credit_bureau_dpd_733P', 'max_credit_bureau_dpdmax_851P', 'mean_credit_bureau_dpdmax_851P', 'median_credit_bureau_dpdmax_851P', 'std_credit_bureau_dpdmax_851P', 'min_credit_bureau_dpdmax_851P', 'count_credit_bureau_dpdmax_851P', 'sum_credit_bureau_dpdmax_851P', 'n_unique_credit_bureau_dpdmax_851P', 'first_credit_bureau_dpdmax_851P', 'last_credit_bureau_dpdmax_851P', 'max_credit_bureau_dpdmaxdatemonth_804T', 'mean_credit_bureau_dpdmaxdatemonth_804T', 'median_credit_bureau_dpdmaxdatemonth_804T', 'std_credit_bureau_dpdmaxdatemonth_804T', 'min_credit_bureau_dpdmaxdatemonth_804T', 'count_credit_bureau_dpdmaxdatemonth_804T', 'sum_credit_bureau_dpdmaxdatemonth_804T', 'n_unique_credit_bureau_dpdmaxdatemonth_804T', 'first_credit_bureau_dpdmaxdatemonth_804T', 'last_credit_bureau_dpdmaxdatemonth_804T', 'max_credit_bureau_dpdmaxdateyear_742T', 'mean_credit_bureau_dpdmaxdateyear_742T', 'median_credit_bureau_dpdmaxdateyear_742T', 'std_credit_bureau_dpdmaxdateyear_742T', 'min_credit_bureau_dpdmaxdateyear_742T', 'count_credit_bureau_dpdmaxdateyear_742T', 'sum_credit_bureau_dpdmaxdateyear_742T', 'n_unique_credit_bureau_dpdmaxdateyear_742T', 'first_credit_bureau_dpdmaxdateyear_742T', 'last_credit_bureau_dpdmaxdateyear_742T', 'max_credit_bureau_installmentamount_644A', 'mean_credit_bureau_installmentamount_644A', 'median_credit_bureau_installmentamount_644A', 'std_credit_bureau_installmentamount_644A', 'min_credit_bureau_installmentamount_644A', 'count_credit_bureau_installmentamount_644A', 'sum_credit_bureau_installmentamount_644A', 'n_unique_credit_bureau_installmentamount_644A', 'first_credit_bureau_installmentamount_644A', 'last_credit_bureau_installmentamount_644A', 'max_credit_bureau_installmentamount_833A', 'mean_credit_bureau_installmentamount_833A', 'median_credit_bureau_installmentamount_833A', 'std_credit_bureau_installmentamount_833A', 'min_credit_bureau_installmentamount_833A', 'count_credit_bureau_installmentamount_833A', 'sum_credit_bureau_installmentamount_833A', 'n_unique_credit_bureau_installmentamount_833A', 'first_credit_bureau_installmentamount_833A', 'last_credit_bureau_installmentamount_833A', 'max_credit_bureau_instlamount_892A', 'mean_credit_bureau_instlamount_892A', 'median_credit_bureau_instlamount_892A', 'std_credit_bureau_instlamount_892A', 'min_credit_bureau_instlamount_892A', 'count_credit_bureau_instlamount_892A', 'sum_credit_bureau_instlamount_892A', 'n_unique_credit_bureau_instlamount_892A', 'first_credit_bureau_instlamount_892A', 'last_credit_bureau_instlamount_892A', 'max_credit_bureau_maxdebtpduevalodued_3940955A', 'mean_credit_bureau_maxdebtpduevalodued_3940955A', 'median_credit_bureau_maxdebtpduevalodued_3940955A', 'std_credit_bureau_maxdebtpduevalodued_3940955A', 'min_credit_bureau_maxdebtpduevalodued_3940955A', 'count_credit_bureau_maxdebtpduevalodued_3940955A', 'sum_credit_bureau_maxdebtpduevalodued_3940955A', 'n_unique_credit_bureau_maxdebtpduevalodued_3940955A', 'first_credit_bureau_maxdebtpduevalodued_3940955A', 'last_credit_bureau_maxdebtpduevalodued_3940955A', 'max_credit_bureau_num_group1', 'mean_credit_bureau_num_group1', 'median_credit_bureau_num_group1', 'std_credit_bureau_num_group1', 'min_credit_bureau_num_group1', 'count_credit_bureau_num_group1', 'sum_credit_bureau_num_group1', 'n_unique_credit_bureau_num_group1', 'first_credit_bureau_num_group1', 'last_credit_bureau_num_group1', 'max_credit_bureau_numberofinstls_810L', 'mean_credit_bureau_numberofinstls_810L', 'median_credit_bureau_numberofinstls_810L', 'std_credit_bureau_numberofinstls_810L', 'min_credit_bureau_numberofinstls_810L', 'count_credit_bureau_numberofinstls_810L', 'sum_credit_bureau_numberofinstls_810L', 'n_unique_credit_bureau_numberofinstls_810L', 'first_credit_bureau_numberofinstls_810L', 'last_credit_bureau_numberofinstls_810L', 'max_credit_bureau_overdueamountmax_950A', 'mean_credit_bureau_overdueamountmax_950A', 'median_credit_bureau_overdueamountmax_950A', 'std_credit_bureau_overdueamountmax_950A', 'min_credit_bureau_overdueamountmax_950A', 'count_credit_bureau_overdueamountmax_950A', 'sum_credit_bureau_overdueamountmax_950A', 'n_unique_credit_bureau_overdueamountmax_950A', 'first_credit_bureau_overdueamountmax_950A', 'last_credit_bureau_overdueamountmax_950A', 'max_credit_bureau_overdueamountmaxdatemonth_494T', 'mean_credit_bureau_overdueamountmaxdatemonth_494T', 'median_credit_bureau_overdueamountmaxdatemonth_494T', 'std_credit_bureau_overdueamountmaxdatemonth_494T', 'min_credit_bureau_overdueamountmaxdatemonth_494T', 'count_credit_bureau_overdueamountmaxdatemonth_494T', 'sum_credit_bureau_overdueamountmaxdatemonth_494T', 'n_unique_credit_bureau_overdueamountmaxdatemonth_494T', 'first_credit_bureau_overdueamountmaxdatemonth_494T', 'last_credit_bureau_overdueamountmaxdatemonth_494T', 'max_credit_bureau_overdueamountmaxdateyear_432T', 'mean_credit_bureau_overdueamountmaxdateyear_432T', 'median_credit_bureau_overdueamountmaxdateyear_432T', 'std_credit_bureau_overdueamountmaxdateyear_432T', 'min_credit_bureau_overdueamountmaxdateyear_432T', 'count_credit_bureau_overdueamountmaxdateyear_432T', 'sum_credit_bureau_overdueamountmaxdateyear_432T', 'n_unique_credit_bureau_overdueamountmaxdateyear_432T', 'first_credit_bureau_overdueamountmaxdateyear_432T', 'last_credit_bureau_overdueamountmaxdateyear_432T', 'max_credit_bureau_pmtdaysoverdue_1135P', 'mean_credit_bureau_pmtdaysoverdue_1135P', 'median_credit_bureau_pmtdaysoverdue_1135P', 'std_credit_bureau_pmtdaysoverdue_1135P', 'min_credit_bureau_pmtdaysoverdue_1135P', 'count_credit_bureau_pmtdaysoverdue_1135P', 'sum_credit_bureau_pmtdaysoverdue_1135P', 'n_unique_credit_bureau_pmtdaysoverdue_1135P', 'first_credit_bureau_pmtdaysoverdue_1135P', 'last_credit_bureau_pmtdaysoverdue_1135P', 'max_credit_bureau_pmtnumpending_403L', 'mean_credit_bureau_pmtnumpending_403L', 'median_credit_bureau_pmtnumpending_403L', 'std_credit_bureau_pmtnumpending_403L', 'min_credit_bureau_pmtnumpending_403L', 'count_credit_bureau_pmtnumpending_403L', 'sum_credit_bureau_pmtnumpending_403L', 'n_unique_credit_bureau_pmtnumpending_403L', 'first_credit_bureau_pmtnumpending_403L', 'last_credit_bureau_pmtnumpending_403L', 'max_credit_bureau_residualamount_1093A', 'mean_credit_bureau_residualamount_1093A', 'median_credit_bureau_residualamount_1093A', 'std_credit_bureau_residualamount_1093A', 'min_credit_bureau_residualamount_1093A', 'count_credit_bureau_residualamount_1093A', 'sum_credit_bureau_residualamount_1093A', 'n_unique_credit_bureau_residualamount_1093A', 'first_credit_bureau_residualamount_1093A', 'last_credit_bureau_residualamount_1093A', 'max_credit_bureau_residualamount_127A', 'mean_credit_bureau_residualamount_127A', 'median_credit_bureau_residualamount_127A', 'std_credit_bureau_residualamount_127A', 'min_credit_bureau_residualamount_127A', 'count_credit_bureau_residualamount_127A', 'sum_credit_bureau_residualamount_127A', 'n_unique_credit_bureau_residualamount_127A', 'first_credit_bureau_residualamount_127A', 'last_credit_bureau_residualamount_127A', 'max_credit_bureau_residualamount_3940956A', 'mean_credit_bureau_residualamount_3940956A', 'median_credit_bureau_residualamount_3940956A', 'std_credit_bureau_residualamount_3940956A', 'min_credit_bureau_residualamount_3940956A', 'count_credit_bureau_residualamount_3940956A', 'sum_credit_bureau_residualamount_3940956A', 'n_unique_credit_bureau_residualamount_3940956A', 'first_credit_bureau_residualamount_3940956A', 'last_credit_bureau_residualamount_3940956A', 'max_credit_bureau_totalamount_503A', 'mean_credit_bureau_totalamount_503A', 'median_credit_bureau_totalamount_503A', 'std_credit_bureau_totalamount_503A', 'min_credit_bureau_totalamount_503A', 'count_credit_bureau_totalamount_503A', 'sum_credit_bureau_totalamount_503A', 'n_unique_credit_bureau_totalamount_503A', 'first_credit_bureau_totalamount_503A', 'last_credit_bureau_totalamount_503A', 'max_credit_bureau_totalamount_881A', 'mean_credit_bureau_totalamount_881A', 'median_credit_bureau_totalamount_881A', 'std_credit_bureau_totalamount_881A', 'min_credit_bureau_totalamount_881A', 'count_credit_bureau_totalamount_881A', 'sum_credit_bureau_totalamount_881A', 'n_unique_credit_bureau_totalamount_881A', 'first_credit_bureau_totalamount_881A', 'last_credit_bureau_totalamount_881A', 'max_credit_bureau_num_group1_right', 'mean_credit_bureau_num_group1_right', 'median_credit_bureau_num_group1_right', 'std_credit_bureau_num_group1_right', 'min_credit_bureau_num_group1_right', 'count_credit_bureau_num_group1_right', 'sum_credit_bureau_num_group1_right', 'n_unique_credit_bureau_num_group1_right', 'first_credit_bureau_num_group1_right', 'last_credit_bureau_num_group1_right', 'max_credit_bureau_num_group2', 'mean_credit_bureau_num_group2', 'median_credit_bureau_num_group2', 'std_credit_bureau_num_group2', 'min_credit_bureau_num_group2', 'count_credit_bureau_num_group2', 'sum_credit_bureau_num_group2', 'n_unique_credit_bureau_num_group2', 'first_credit_bureau_num_group2', 'last_credit_bureau_num_group2', 'max_credit_bureau_pmts_dpdvalue_108P', 'mean_credit_bureau_pmts_dpdvalue_108P', 'median_credit_bureau_pmts_dpdvalue_108P', 'std_credit_bureau_pmts_dpdvalue_108P', 'min_credit_bureau_pmts_dpdvalue_108P', 'count_credit_bureau_pmts_dpdvalue_108P', 'sum_credit_bureau_pmts_dpdvalue_108P', 'n_unique_credit_bureau_pmts_dpdvalue_108P', 'first_credit_bureau_pmts_dpdvalue_108P', 'last_credit_bureau_pmts_dpdvalue_108P', 'max_credit_bureau_pmts_pmtsoverdue_635A', 'mean_credit_bureau_pmts_pmtsoverdue_635A', 'median_credit_bureau_pmts_pmtsoverdue_635A', 'std_credit_bureau_pmts_pmtsoverdue_635A', 'min_credit_bureau_pmts_pmtsoverdue_635A', 'count_credit_bureau_pmts_pmtsoverdue_635A', 'sum_credit_bureau_pmts_pmtsoverdue_635A', 'n_unique_credit_bureau_pmts_pmtsoverdue_635A', 'first_credit_bureau_pmts_pmtsoverdue_635A', 'last_credit_bureau_pmts_pmtsoverdue_635A', 'std_debitcard_num_group1', 'min_debitcard_num_group1', 'std_deposit_amount_416A', 'std_deposit_num_group1', 'min_deposit_num_group1', 'min_person2_num_group2', 'first_person2_num_group2', 'cardtype_51L', 'cntincpaycont9m_3716944L', 'cntpmts24_3658933L', 'commnoinclast6m_3546845L', 'datefirstoffer_1144D', 'datelastinstal40dpd_247D', 'datelastunpaid_3546854D', 'daysoverduetolerancedd_3976961L', 'deferredmnthsnum_166L', 'dtlastpmtallstes_4499206D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'firstclxcampaign_1125D', 'firstdatedue_489D', 'interestrategrace_34L', 'isbidproductrequest_292L', 'lastactivateddate_801D', 'lastapplicationdate_877D', 'lastapprcommoditycat_1041M', 'lastapprcommoditytypec_5251766M', 'lastapprdate_640D', 'lastcancelreason_561M', 'lastdelinqdate_224D', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrejectcommoditycat_161M', 'lastrejectcommodtypec_5251769M', 'lastrejectdate_50D', 'lastrejectreason_759M', 'lastrejectreasonclient_4145040M', 'lastrepayingdate_696D', 'lastst_736L', 'mastercontrelectronic_519L', 'mastercontrexist_109L', 'maxdpdinstldate_3546855D', 'monthsannuity_845L', 'numincomingpmts_3546848L', 'numinstlallpaidearly3d_817L', 'numinstlsallpaid_934L', 'numinstlswithdpd10_728L', 'numinstlswithdpd5_4187116L', 'numinstlswithoutdpd_562L', 'numinstmatpaidtearly2d_4499204L', 'numinstpaid_4499208L', 'numinstpaidearly3d_3546850L', 'numinstpaidearly3dest_4493216L', 'numinstpaidearly5d_1087L', 'numinstpaidearly5dest_4493211L', 'numinstpaidearly5dobd_4499205L', 'numinstpaidearly_338L', 'numinstpaidearlyest_4493214L', 'numinstpaidlastcontr_4325080L', 'numinstpaidlate1d_3546852L', 'numinstregularpaid_973L', 'numinstregularpaidest_4493210L', 'numinsttopaygr_769L', 'numinsttopaygrest_4493213L', 'numinstunpaidmax_3546851L', 'numinstunpaidmaxest_4493212L', 'payvacationpostpone_4187118D', 'pctinstlsallpaidearl3d_427L', 'pctinstlsallpaidlat10d_839L', 'pctinstlsallpaidlate1d_3546856L', 'pctinstlsallpaidlate4d_3546849L', 'pctinstlsallpaidlate6d_3546844L', 'previouscontdistrict_112M', 'validfrom_1069D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'contractssum_5085716L', 'dateofbirth_337D', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'pmtscount_423L', 'pmtssum_45A', 'responsedate_4527233D', 'responsedate_4917613D', 'riskassesment_302T', 'riskassesment_940T', 'min_tax_num_group1', 'max_tax_amount_4917619A', 'mean_tax_amount_4917619A', 'median_tax_amount_4917619A', 'std_tax_amount_4917619A', 'min_tax_amount_4917619A', 'first_tax_amount_4917619A', 'last_tax_amount_4917619A', 'credacc_status_367L_0', 'credtype_587L_2', 'education_1138M_2', 'familystate_726L_2', 'inittransactioncode_279L_2', 'postype_4733339M_1', 'cacccardblochreas_147M_0', 'conts_type_509L_1', 'credacc_cards_status_52L_0', 'bankacctype_710L_0', 'clientscnt_136L_0', 'credtype_322L_3', 'disbursementtype_67L_3', 'inittransactioncode_186L_3', 'isdebitcard_729L_0', 'paytype1st_925L_1', 'paytype_783L_1', 'typesuite_864L_1', 'assignmentdate_238D_0', 'birthdate_574D_0', 'education_1103M_5', 'education_88M_5', 'maritalst_385M_6', 'maritalst_893M_6', 'requesttype_4525192L_3', 'responsedate_1012D_0', 'case_id', 'WEEK_NUM']\n\nprint(f\"len(drop_cols):{len(drop_cols)},drop_cols:{drop_cols}\")\ntrain_feats=train_feats.drop(drop_cols,axis=1)\ntest_feats=test_feats.drop(drop_cols,axis=1)\n\nprint(\"----------fillna value ****\")\ntrain_feats.fillna(-1,inplace=True)\ntest_feats.fillna(-1,inplace=True)\n\nprint(f\"len(drop_cols):{len(drop_cols)},total_features_count:{len(test_feats.columns)}\")\n\nprint(\"----------drop corr>0.99 cols ****\")\n#我们这里就是找相关性特别高的特征对,可以考虑对它们进行降维操作.\n#计算两组变量的皮尔逊相关系数\ndef pearson_corr(x1,x2):\n    \"\"\"\n    x1,x2:np.array\n    \"\"\"\n    mean_x1=np.mean(x1)\n    mean_x2=np.mean(x2)\n    std_x1=np.std(x1)\n    std_x2=np.std(x2)\n    pearson=np.mean((x1-mean_x1)*(x2-mean_x2))/(std_x1*std_x2)\n    return pearson\ndef find_corr():\n    cols=train_feats.columns\n    drop_cols=[]#2个特征相关性系数非常接近1了,所以会考虑drop掉一个特征\n    corr_cols=[]#比如第1,3,4个特征它们的相关性特别高,我们会考虑[[1,3,4]],然后降维\n    ignore_cols=[]#在遍历过程中出现在idx后面的特征,可能已经和idx前面的某个特征相关性高,需要做降维了,所以不需要再计算相关性了\n    for idx in range(len(cols)):\n        if cols[idx] in ignore_cols:\n            continue\n        corr_col=[cols[idx]]#这里和idx相关的特征\n        for j in range(idx+1,len(cols)):\n            if cols[j] in ignore_cols:\n                continue\n            #计算两组特征的皮尔逊相关系数\n            pearson=pearson_corr(train_feats[cols[idx]].values,train_feats[cols[j]].values)\n            if abs(pearson)>0.99:#相关性太高了,直接去掉一个特征\n                drop_cols.append(cols[j])\n                ignore_cols.append(cols[j])\n                print(f\"the {cols[idx]} and {cols[j]} pearson_corr is {pearson}\")\n            elif abs(pearson)>0.8:#如果皮尔逊相关系数一般高\n                corr_col.append(cols[j])\n                ignore_cols.append(cols[j])\n                print(f\"the {cols[idx]} and {cols[j]} pearson_corr is {pearson}\")\n        if len(corr_col)>1:#因为已经有cols[idx],大于1就是有其他相关性特征.\n            corr_cols.append(corr_col)\n            print(\"-\"*50)\n    #因为和某些特征相关性为1,所以去掉的列名\n    print(f\"drop_cols:{drop_cols},corr_cols:{corr_cols}\")\n    print(f\"pearson_corr_pairs_counts:{len(corr_cols)}\")\n    return drop_cols,corr_cols\n# #每次加入新的文件都要重新找新的drop_cols和corr_cols\n# drop_cols,corr_cols=find_corr()\n\ndrop_cols=['tenor_203L', 'sum_deposit_num_group1', 'last_person2_num_group1', 'first_person2_num_group1', 'avgdbdtollast24m_4525197P', 'interestrate_311L', 'numberofqueries_373L', 'max_tax_num_group1', 'mean_tax_num_group1', 'median_tax_num_group1', 'std_tax_num_group1', 'count_tax_num_group1', 'n_unique_tax_num_group1', 'inittransactioncode_279L_0', 'inittransactioncode_186L_1']\ncorr_cols=[['isbidproduct_390L', 'opencred_647L', 'cacccardblochreas_147M_1'], ['num_group1', 'num_group1_right'], ['max_debitcard_num_group1', 'mean_debitcard_num_group1', 'median_debitcard_num_group1', 'count_debitcard_num_group1', 'n_unique_debitcard_num_group1', 'first_debitcard_num_group1', 'last_debitcard_num_group1', 'count_deposit_amount_416A', 'n_unique_deposit_amount_416A', 'max_deposit_num_group1', 'mean_deposit_num_group1', 'median_deposit_num_group1', 'count_deposit_num_group1', 'n_unique_deposit_num_group1', 'first_deposit_num_group1', 'last_deposit_num_group1'], ['max_deposit_amount_416A', 'mean_deposit_amount_416A', 'median_deposit_amount_416A', 'sum_deposit_amount_416A', 'first_deposit_amount_416A'], ['min_deposit_amount_416A', 'last_deposit_amount_416A'], ['max_person2_num_group1', 'mean_person2_num_group1', 'median_person2_num_group1', 'count_person2_num_group1', 'n_unique_person2_num_group1', 'count_person2_num_group2'], ['std_person2_num_group1', 'std_person2_num_group2'], ['min_person2_num_group1', 'mean_person2_num_group2', 'median_person2_num_group2', 'n_unique_person2_num_group2', 'paytype1st_925L_0', 'paytype_783L_0'], ['sum_person2_num_group1', 'max_person2_num_group2', 'last_person2_num_group2'], ['amtinstpaidbefduel24m_4187115A', 'maxdebt4_972A'], ['annuity_780A', 'credamount_770A'], ['applicationscnt_464L', 'applicationscnt_629L'], ['applicationscnt_867L', 'sellerplacescnt_216L'], ['avgdbddpdlast24m_3658932P', 'avgdpdtolclosure24_3658938P', 'mindbddpdlast24m_3658935P', 'mindbdtollast24m_4525191P'], ['avgdbddpdlast3m_4187120P', 'maxdbddpdlast1m_3658939P', 'maxdbddpdtollast12m_3658940P', 'maxdbddpdtollast6m_4187119P'], ['avgoutstandbalancel6m_4187114A', 'currdebt_22A', 'maxoutstandbalancel12m_4187113A', 'sumoutstandtotal_3546847A', 'sumoutstandtotalest_4493215A', 'totaldebt_9A'], ['clientscnt12m_3712952L', 'clientscnt3m_3712950L', 'clientscnt6m_3712949L', 'clientscnt_1022L'], ['eir_270L', 'credtype_322L_1', 'isdebitcard_729L_1'], ['maxdpdfrom6mto36m_3546853P', 'maxdpdlast24m_143P', 'maxdpdtolerance_374P'], ['maxdpdlast12m_727P', 'maxdpdlast9m_1059P'], ['maxdpdlast3m_392P', 'maxdpdlast6m_474P'], ['posfpd10lastmonth_333P', 'posfpd30lastmonth_3976960P'], ['days120_123L', 'days180_256L', 'days90_310L'], ['max_tax_amount_4527230A', 'mean_tax_amount_4527230A', 'median_tax_amount_4527230A', 'std_tax_amount_4527230A', 'sum_tax_amount_4527230A', 'first_tax_amount_4527230A', 'last_tax_amount_4527230A'], ['count_tax_amount_4527230A', 'n_unique_tax_amount_4527230A', 'first_tax_num_group1', 'last_tax_num_group1'], ['count_tax_amount_4917619A', 'n_unique_tax_amount_4917619A'], ['credtype_587L_1', 'inittransactioncode_279L_1'], ['bankacctype_710L_1', 'credtype_322L_0', 'disbursementtype_67L_0', 'disbursementtype_67L_2', 'inittransactioncode_186L_0', 'inittransactioncode_186L_2'], ['education_88M_0', 'education_88M_1', 'maritalst_893M_0']]\n\ntrain_feats=train_feats.drop(drop_cols,axis=1)\ntest_feats=test_feats.drop(drop_cols,axis=1)\n\nprint(\"----------SVD decomposition ****\")\n\nfor idx in range(len(corr_cols)):\n    #这几列相关性比较强\n    corr_col=corr_cols[idx]\n    train_X=train_feats[corr_col].values\n    test_X=test_feats[corr_col].values\n    #创建截断奇异值分解的对象,降维到原数据一半的维度\n    svd=TruncatedSVD(n_components=len(train_X[0])//2,random_state=Config.seed)#降维就降成原来维度的一半吧\n    svd.fit(train_X)\n    svd_train=svd.transform(train_X)\n    svd_test=svd.transform(test_X)\n\n    for j in range(len(svd_test[0])):\n        train_feats[f'svd_{idx}_{j}']=svd_train[:,j]\n        test_feats[f'svd_{idx}_{j}']=svd_test[:,j]\n    train_feats.drop(corr_col,axis=1,inplace=True)\n    test_feats.drop(corr_col,axis=1,inplace=True)\n\nprint(f\"len(drop_cols):{len(drop_cols)},total_features_count:{len(test_feats.columns)}\")\ntrain_feats.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reduce memory","metadata":{}},{"cell_type":"code","source":"#遍历表格df的所有列修改数据类型减少内存使用\ndef reduce_mem_usage(df, float16_as32=True):\n    #memory_usage()是df每列的内存使用量,sum是对它们求和, B->KB->MB\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:#遍历每列的列名\n        col_type = df[col].dtype#列名的type\n        if col_type != object:#不是object也就是说这里处理的是数值类型的变量\n            c_min,c_max = df[col].min(),df[col].max() #求出这列的最大值和最小值\n            if str(col_type)[:3] == 'int':#如果是int类型的变量,不管是int8,int16,int32还是int64\n                #如果这列的取值范围是在int8的取值范围内,那就对类型进行转换 (-128 到 127)\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                #如果这列的取值范围是在int16的取值范围内,那就对类型进行转换(-32,768 到 32,767)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                #如果这列的取值范围是在int32的取值范围内,那就对类型进行转换(-2,147,483,648到2,147,483,647)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                #如果这列的取值范围是在int64的取值范围内,那就对类型进行转换(-9,223,372,036,854,775,808到9,223,372,036,854,775,807)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:#如果是浮点数类型.\n                #如果数值在float16的取值范围内,如果觉得需要更高精度可以考虑float32\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32:#如果数据需要更高的精度可以选择float32\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float16)  \n                #如果数值在float32的取值范围内，对它进行类型转换\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                #如果数值在float64的取值范围内，对它进行类型转换\n                else:\n                    df[col] = df[col].astype(np.float64)\n    #计算一下结束后的内存\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    #相比一开始的内存减少了百分之多少\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\ntrain_feats = reduce_mem_usage(train_feats)\ntest_feats = reduce_mem_usage(test_feats)\ngc.collect()#手动触发垃圾回收,强制回收由垃圾回收器标记为未使用的内存","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model training","metadata":{}},{"cell_type":"code","source":"N_THREADS = 4\nN_FOLDS = 10\nTEST_SIZE = 0.1\nTIMEOUT = 5 * 3600\n#保存训练好的树模型,obj是保存的模型,path是需要保存的路径\ndef pickle_dump(obj, path):\n    #打开指定的路径path,binary write(二进制写入)\n    with open(path, mode=\"wb\") as f:\n        #将obj对象保存到f,使用协议版本4进行序列化\n        dill.dump(obj, f, protocol=4)\n#任务是回归任务,损失是MSE,评估指标是RMSE\ntask = Task('binary', loss = 'logloss', metric =roc_auc_score)\nroles = {\n    'target': Config.TARGET_NAME\n}\nautoml = TabularAutoML(\n    task = task, \n    timeout = TIMEOUT,\n    cpu_limit = N_THREADS,\n    reader_params = {'n_jobs': N_THREADS, 'cv': N_FOLDS, 'random_state': Config.seed, 'advanced_roles': False}\n)\n#%%time \noof_pred = automl.fit_predict(train_feats, roles = roles, verbose = 3)\nprint(automl.create_model_str_desc())\npickle_dump(automl, f'automl.model') #保存训练好的模型","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"test_preds=automl.predict(test_feats).data.T[0]\nprint(\"trick is all you need\")\n#如果模型的效果会随着时间的推移越变越差\n#比如 [0.9,0.8,0.7,0.6] 那我们需要将0.9变成0.7,0.8也变成0.7,所以每个预测的结果减小的值应该不同.\nchange_range=len(test_preds)//2#对前半部分的预测效果做微调\nchange_value=0.01#效果变差多少\nfor idx in range(change_range):#idx越大,模型效果本身就越差,所以微调的程度越小.\n    test_preds[idx]-=change_value*(1-idx/change_range)\nsubmission=pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\nsubmission['score']=np.clip(np.nan_to_num(test_preds,nan=0.3),0,1)\nsubmission.to_csv(\"submission.csv\",index=None)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}