{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"dataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-27T03:59:40.944767Z","iopub.execute_input":"2024-06-27T03:59:40.948308Z","iopub.status.idle":"2024-06-27T03:59:40.964011Z","shell.execute_reply.started":"2024-06-27T03:59:40.948231Z","shell.execute_reply":"2024-06-27T03:59:40.958641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nfor dirname, _, filenames in os.walk(dataPath+'csv_files/test/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T03:59:42.338537Z","iopub.execute_input":"2024-06-27T03:59:42.338960Z","iopub.status.idle":"2024-06-27T03:59:42.946083Z","shell.execute_reply.started":"2024-06-27T03:59:42.338925Z","shell.execute_reply":"2024-06-27T03:59:42.943884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport lightgbm as lgb \nimport numpy as np \nimport pandas as pd  \n#没有pl\n##import polars as pl  \nimport warnings\nfrom catboost import CatBoostClassifier, Pool \nfrom glob import glob\nfrom IPython.display import display \nfrom pathlib import Path\nfrom sklearn.base import BaseEstimator, ClassifierMixin \nfrom sklearn.metrics import roc_auc_score \n#没有StratifiedGroupKFold\n##from sklearn.model_selection import StratifiedGroupKFold  # type: ignore\nfrom typing import Any\n\nwarnings.filterwarnings(\"ignore\")\n\nROOT = Path(dataPath+\"csv_files/test\")\nTRAIN_DIR = ROOT / \"csv\" / \"train\"\nTEST_DIR = ROOT / \"csv\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-06-27T03:59:45.502539Z","iopub.execute_input":"2024-06-27T03:59:45.504421Z","iopub.status.idle":"2024-06-27T03:59:47.236727Z","shell.execute_reply.started":"2024-06-27T03:59:45.504368Z","shell.execute_reply":"2024-06-27T03:59:47.235089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df[col] = df[col].astype('int32')\n            elif col in [\"date_decision\"]:\n                df[col] = df[col].astype('datetime64[ns]')\n            elif col[-1] in (\"P\", \"A\"):\n                df[col] = df[col].astype('float')\n            elif col[-1] in (\"M\",):\n                df[col] = df[col].astype('str')\n            elif col[-1] in (\"D\",):\n                df[col] = df[col].astype('datetime64[ns]')         \n\n        return df\n\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df[col] = pd.DatetimeIndex(df[col]) - pd.DatetimeIndex(df['date_decision'])\n                # 将'col'列转换为总天数\n                df[col] = df[col].dt.days\n                # 将'col'列转换为Float32类型\n                df[col] = df[col].astype('float32')\n                \n        df = df.drop(\"date_decision\",axis=1)\n        df = df.drop(\"MONTH\",axis=1)\n        return df\n\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].isnull().mean()\n\n                if isnull > 0.3:\n                    df = df.drop(col,axis=1)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pd.StringDtype()):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col,axis=1)\n\n        return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pd.read_parquet(path).query('num_group1 == 0')\n        df = df.pipe(Pipeline.set_table_dtypes)    \n        chunks.append(df)\n        \n    df = pd.concat(chunks, axis=0)\n    return df\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-06-27T03:59:50.072493Z","iopub.execute_input":"2024-06-27T03:59:50.073759Z","iopub.status.idle":"2024-06-27T03:59:50.089430Z","shell.execute_reply.started":"2024-06-27T03:59:50.073715Z","shell.execute_reply":"2024-06-27T03:59:50.087993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pd.read_csv(dataPath+'csv_files/train/train_base.csv').pipe(Pipeline.set_table_dtypes)\n\n# 使用assign函数创建新的列\ntrain_basetable = train_basetable.assign(\n    month_decision=train_basetable['date_decision'].dt.month,\n    weekday_decision=train_basetable['date_decision'].dt.weekday\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 =pd.read_csv(dataPath+'csv_files/train/train_static_0_0.csv').pipe(Pipeline.set_table_dtypes)\ndf2 =pd.read_csv(dataPath+'csv_files/train/train_static_0_1.csv').pipe(Pipeline.set_table_dtypes)\n\ntrain_static_0 = pd.concat([df1, df2], axis=0).pipe(Pipeline.filter_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_cb=pd.read_csv(dataPath+'csv_files/train/train_static_cb_0.csv').pipe(Pipeline.set_table_dtypes).pipe(Pipeline.filter_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1=pd.read_csv(dataPath+'csv_files/train/train_person_1.csv')\ntrain_person_1=train_person_1[train_person_1['num_group1'] == 0].pipe(Pipeline.set_table_dtypes).pipe(Pipeline.filter_cols)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person1_feats=train_person_1[['case_id','birth_259D','mainoccupationinc_384A']].pipe(Pipeline.set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfolder_path =dataPath+ 'csv_files/train' \nfile_names = os.listdir(folder_path)\n\nbureau_a_1 = pd.DataFrame()\n\nfor file_name in file_names:\n    if file_name.startswith('train_credit_bureau_a_1_') and file_name.endswith('.csv'):\n        file_path = os.path.join(folder_path, file_name)\n        temp_df = pd.read_csv(file_path, usecols=['case_id', 'financialinstitution_591M'])\n        bureau_a_1 = pd.concat([bureau_a_1, temp_df], ignore_index=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau= pd.concat([bureau_a_1,pd.read_csv(dataPath+'csv_files/train/train_credit_bureau_b_1.csv')\\\n.pipe(Pipeline.set_table_dtypes).rename(columns={\"credor_3940957M\": \"financialinstitution_591M\"}, inplace=True)], axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau=bureau.drop_duplicates()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"taxes=pd.concat([\npd.read_csv(dataPath+'csv_files/train/train_tax_registry_a_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4527230A': 'sum','recorddate_4527225D': 'last'}).rename(columns={\"amount_4527230A\": \"amount_taxA\",\"recorddate_4527225D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/train/train_tax_registry_b_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4917619A': 'sum', 'deductiondate_4917603D': 'last'}).rename(columns={\"amount_4917619A\": \"amount_taxA\",\"deductiondate_4917603D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/train/train_tax_registry_c_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'pmtamount_36A': 'sum', 'processingdate_168D': 'last'}).rename(columns={\"pmtamount_36A\": \"amount_taxA\",\"processingdate_168D\":\"recorddate_taxD\"})]\n, axis=0).pipe(Pipeline.filter_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_person_1\ndel bureau_a_1\ndel df1\ndel df2\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0=train_static_0.set_index(['case_id'])\ntrain_static_cb=train_static_cb.set_index(['case_id'])\ntrain_person1_feats=train_person1_feats.set_index(['case_id'])\nbureau=bureau.set_index(['case_id'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes= train_basetable.join(train_static_0, how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb, how=\"left\", on=\"case_id\"\n).join(\n    train_person1_feats, how=\"left\", on=\"case_id\"\n).join(\n   bureau, how=\"left\", on=\"case_id\"\n).join(\n   taxes, how=\"left\", on=\"case_id\"\n).pipe(Pipeline.handle_dates)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes = bureau_taxes.drop_duplicates('case_id', keep='first')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=bureau_taxes.drop('dateofbirth_337D',axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df_train['target']==1).mean()*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"建模","metadata":{}},{"cell_type":"code","source":"del train_static_0\ndel train_static_cb\ndel train_person1_feats\ndel bureau\ndel taxes\ndel bureau_taxes\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nfrom sklearn.model_selection import train_test_split\nimport xgboost as xgb\nfrom xgboost import plot_importance\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport pandas as pd\nimport numpy as np\nimport warnings\nfrom xgboost.sklearn import XGBClassifier\nfrom sklearn import metrics","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=df_train.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=data.drop(columns=['credtype_322L','disbursementtype_67L','inittransactioncode_186L','lastapprcommoditycat_1041M','lastapprcommoditytypec_5251766M','lastcancelreason_561M','lastrejectcommoditycat_161M','lastrejectcommodtypec_5251769M','lastrejectreason_759M','lastrejectreasonclient_4145040M','lastst_736L','opencred_647L','paytype1st_925L','paytype_783L','previouscontdistrict_112M','twobodfilling_608L','description_5085714M','education_1103M','education_88M','maritalst_385M','maritalst_893M','financialinstitution_591M'],axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = data.drop(columns=['case_id', 'WEEK_NUM', 'target'])\ny = data['target']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 划分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ndel data\ndel y_test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(\nbooster='gbtree',\nlearning_rate =0.1,\nn_estimators=10,\nreg_alpha=0.1,\nmax_depth=2,\nmin_child_weight=5,\ngamma=0.1,\nsubsample=0.9,\ncolsample_bytree=0.7,\nnthread=4,\nscale_pos_weight=1,\nseed=27) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_all=model.fit(X_train,y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model_all.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ndel y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"预测结果","metadata":{}},{"cell_type":"code","source":"folder_path = dataPath+'csv_files/test' \nfile_names = os.listdir(folder_path)\n\ntest_bureau_a_1 = pd.DataFrame()\n\nfor file_name in file_names:\n    if file_name.startswith('test_credit_bureau_a_1_') and file_name.endswith('.csv'):\n        file_path = os.path.join(folder_path, file_name)\n        temp_df = pd.read_csv(file_path, usecols=['case_id', 'financialinstitution_591M'])\n        test_bureau_a_1 = pd.concat([test_bureau_a_1, temp_df], ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_bureau= pd.concat([test_bureau_a_1,pd.read_csv(dataPath+'csv_files/test/test_credit_bureau_b_1.csv')\\\n.pipe(Pipeline.set_table_dtypes).rename(columns={\"credor_3940957M\": \"financialinstitution_591M\"})[['case_id','financialinstitution_591M']]], axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pd.read_csv(dataPath+'csv_files/test/test_base.csv').pipe(Pipeline.set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = test_basetable.assign(\n    month_decision=test_basetable['date_decision'].dt.month,\n    weekday_decision=test_basetable['date_decision'].dt.weekday\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3 =pd.read_csv(dataPath+'csv_files/test/test_static_0_0.csv').pipe(Pipeline.set_table_dtypes)\ndf4 =pd.read_csv(dataPath+'csv_files/test/test_static_0_1.csv').pipe(Pipeline.set_table_dtypes)\ndf5 =pd.read_csv(dataPath+'csv_files/test/test_static_0_2.csv').pipe(Pipeline.set_table_dtypes)\n\ntest_static_0 = pd.concat([df3, df4,df5], axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static_cb=pd.read_csv(dataPath+'csv_files/test/test_static_cb_0.csv').pipe(Pipeline.set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1=pd.read_csv(dataPath+'csv_files/test/test_person_1.csv')\ntest_person_1=test_person_1[test_person_1['num_group1'] == 0].pipe(Pipeline.set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person1_feats=test_person_1[['case_id','birth_259D','mainoccupationinc_384A']].pipe(Pipeline.set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_taxes=pd.concat([\npd.read_csv(dataPath+'csv_files/test/test_tax_registry_a_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4527230A': 'sum','recorddate_4527225D': 'last'}).rename(columns={\"amount_4527230A\": \"amount_taxA\",\"recorddate_4527225D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/test/test_tax_registry_b_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4917619A': 'sum', 'deductiondate_4917603D': 'last'}).rename(columns={\"amount_4917619A\": \"amount_taxA\",\"deductiondate_4917603D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/test/test_tax_registry_c_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'pmtamount_36A': 'sum', 'processingdate_168D': 'last'}).rename(columns={\"pmtamount_36A\": \"amount_taxA\",\"processingdate_168D\":\"recorddate_taxD\"})]\n, axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes_test= test_basetable.merge(test_static_0, how=\"left\", on=\"case_id\"\n).merge(\n    test_static_cb, how=\"left\", on=\"case_id\"\n ).merge(\n    test_person1_feats, how=\"left\", on=\"case_id\"\n).merge(\n   test_bureau, how=\"left\", on=\"case_id\"\n ).merge(\n   test_taxes, how=\"left\", on=\"case_id\"\n).pipe(Pipeline.handle_dates)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes_test = bureau_taxes_test.drop_duplicates('case_id', keep='first')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test=bureau_taxes_test.drop('dateofbirth_337D',axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.set_index('case_id', inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a=list(X_train.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"预测结果","metadata":{}},{"cell_type":"code","source":"xgb_pred = pd.Series(model_all.predict_proba(df_test[a])[:, 1], index=df_test[a].index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = pd.read_csv(dataPath+\"sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = pd.DataFrame({\n    \"case_id\": df_subm[\"case_id\"].to_numpy(),\n    \"score\": xgb_pred\n}).set_index('case_id')\ndf_subm.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}