{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"dataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.484880Z","iopub.execute_input":"2024-10-06T09:44:22.486430Z","iopub.status.idle":"2024-10-06T09:44:22.504206Z","shell.execute_reply.started":"2024-10-06T09:44:22.486361Z","shell.execute_reply":"2024-10-06T09:44:22.502306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nfor dirname, _, filenames in os.walk(dataPath+'csv_files/test/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\nimport gc\nimport lightgbm as lgb \nimport numpy as np \nimport pandas as pd  \n#没有pl\n##import polars as pl  \nimport warnings\nfrom catboost import CatBoostClassifier, Pool \nfrom glob import glob\nfrom IPython.display import display \nfrom pathlib import Path\nfrom sklearn.base import BaseEstimator, ClassifierMixin \nfrom sklearn.metrics import roc_auc_score \n#没有StratifiedGroupKFold\n##from sklearn.model_selection import StratifiedGroupKFold  # type: ignore\nfrom typing import Any\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.508529Z","iopub.execute_input":"2024-10-06T09:44:22.509157Z","iopub.status.idle":"2024-10-06T09:44:22.560115Z","shell.execute_reply.started":"2024-10-06T09:44:22.509096Z","shell.execute_reply":"2024-10-06T09:44:22.558756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = Path(dataPath+\"csv_files/test\")\nTRAIN_DIR = ROOT / \"csv\" / \"train\"\nTEST_DIR = ROOT / \"csv\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.561940Z","iopub.execute_input":"2024-10-06T09:44:22.562400Z","iopub.status.idle":"2024-10-06T09:44:22.570031Z","shell.execute_reply.started":"2024-10-06T09:44:22.562324Z","shell.execute_reply":"2024-10-06T09:44:22.568682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df[col] = df[col].astype('int32')\n            elif col in [\"date_decision\"]:\n                df[col] = df[col].astype('datetime64[ns]')\n            elif col[-1] in (\"P\", \"A\"):\n                df[col] = df[col].astype('float')\n            elif col[-1] in (\"M\",):\n                df[col] = df[col].astype('str')\n            elif col[-1] in (\"D\",):\n                df[col] = df[col].astype('datetime64[ns]')         \n\n        return df\n\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df[col] = pd.DatetimeIndex(df[col]) - pd.DatetimeIndex(df['date_decision'])\n                # 将'col'列转换为总天数\n                df[col] = df[col].dt.days\n                # 将'col'列转换为Float32类型\n                df[col] = df[col].astype('float32')\n                \n        df = df.drop(\"date_decision\",axis=1)\n        df = df.drop(\"MONTH\",axis=1)\n        return df\n\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].isnull().mean()\n\n                if isnull > 0.3:\n                    df = df.drop(col,axis=1)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pd.StringDtype()):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col,axis=1)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.571873Z","iopub.execute_input":"2024-10-06T09:44:22.572372Z","iopub.status.idle":"2024-10-06T09:44:22.599297Z","shell.execute_reply.started":"2024-10-06T09:44:22.572301Z","shell.execute_reply":"2024-10-06T09:44:22.597553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pd.read_parquet(path).query('num_group1 == 0')\n        df = df.pipe(Pipeline.set_table_dtypes)    \n        chunks.append(df)\n        \n    df = pd.concat(chunks, axis=0)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.604746Z","iopub.execute_input":"2024-10-06T09:44:22.605544Z","iopub.status.idle":"2024-10-06T09:44:22.617376Z","shell.execute_reply.started":"2024-10-06T09:44:22.605478Z","shell.execute_reply":"2024-10-06T09:44:22.615031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.619760Z","iopub.execute_input":"2024-10-06T09:44:22.620423Z","iopub.status.idle":"2024-10-06T09:44:22.637791Z","shell.execute_reply.started":"2024-10-06T09:44:22.620362Z","shell.execute_reply":"2024-10-06T09:44:22.636222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_basetable = pd.read_csv(dataPath+'csv_files/train/train_base.csv').pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:22.640452Z","iopub.execute_input":"2024-10-06T09:44:22.640995Z","iopub.status.idle":"2024-10-06T09:44:25.203152Z","shell.execute_reply.started":"2024-10-06T09:44:22.640945Z","shell.execute_reply":"2024-10-06T09:44:25.201781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 使用assign函数创建新的列\ntrain_basetable = train_basetable.assign(\n    month_decision=train_basetable['date_decision'].dt.month,\n    weekday_decision=train_basetable['date_decision'].dt.weekday\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:25.205270Z","iopub.execute_input":"2024-10-06T09:44:25.205742Z","iopub.status.idle":"2024-10-06T09:44:25.344430Z","shell.execute_reply.started":"2024-10-06T09:44:25.205697Z","shell.execute_reply":"2024-10-06T09:44:25.343180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 =pd.read_csv(dataPath+'csv_files/train/train_static_0_0.csv').pipe(Pipeline.set_table_dtypes)\ndf2 =pd.read_csv(dataPath+'csv_files/train/train_static_0_1.csv').pipe(Pipeline.set_table_dtypes)\n\ntrain_static_0 = pd.concat([df1, df2], axis=0).pipe(Pipeline.filter_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:44:25.346078Z","iopub.execute_input":"2024-10-06T09:44:25.346487Z","iopub.status.idle":"2024-10-06T09:46:34.149279Z","shell.execute_reply.started":"2024-10-06T09:44:25.346446Z","shell.execute_reply":"2024-10-06T09:46:34.147582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_cb=pd.read_csv(dataPath+'csv_files/train/train_static_cb_0.csv').pipe(Pipeline.set_table_dtypes).pipe(Pipeline.filter_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:46:34.151149Z","iopub.execute_input":"2024-10-06T09:46:34.151614Z","iopub.status.idle":"2024-10-06T09:46:57.479876Z","shell.execute_reply.started":"2024-10-06T09:46:34.151568Z","shell.execute_reply":"2024-10-06T09:46:57.478571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1=pd.read_csv(dataPath+'csv_files/train/train_person_1.csv')\ntrain_person_1=train_person_1[train_person_1['num_group1'] == 0].pipe(Pipeline.set_table_dtypes).pipe(Pipeline.filter_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:46:57.481494Z","iopub.execute_input":"2024-10-06T09:46:57.481906Z","iopub.status.idle":"2024-10-06T09:47:31.962418Z","shell.execute_reply.started":"2024-10-06T09:46:57.481867Z","shell.execute_reply":"2024-10-06T09:47:31.961107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person1_feats=train_person_1[['case_id','birth_259D','mainoccupationinc_384A']].pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:47:31.964314Z","iopub.execute_input":"2024-10-06T09:47:31.964835Z","iopub.status.idle":"2024-10-06T09:47:32.002141Z","shell.execute_reply.started":"2024-10-06T09:47:31.964778Z","shell.execute_reply":"2024-10-06T09:47:32.000951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfolder_path =dataPath+ 'csv_files/train' \nfile_names = os.listdir(folder_path)\n\nbureau_a_1 = pd.DataFrame()\n\nfor file_name in file_names:\n    if file_name.startswith('train_credit_bureau_a_1_') and file_name.endswith('.csv'):\n        file_path = os.path.join(folder_path, file_name)\n        temp_df = pd.read_csv(file_path, usecols=['case_id', 'financialinstitution_591M'])\n        bureau_a_1 = pd.concat([bureau_a_1, temp_df], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:47:32.003763Z","iopub.execute_input":"2024-10-06T09:47:32.004190Z","iopub.status.idle":"2024-10-06T09:49:20.552598Z","shell.execute_reply.started":"2024-10-06T09:47:32.004148Z","shell.execute_reply":"2024-10-06T09:49:20.551099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau= pd.concat([bureau_a_1,pd.read_csv(dataPath+'csv_files/train/train_credit_bureau_b_1.csv')\\\n.pipe(Pipeline.set_table_dtypes).rename(columns={\"credor_3940957M\": \"financialinstitution_591M\"}, inplace=True)], axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:49:20.559013Z","iopub.execute_input":"2024-10-06T09:49:20.559495Z","iopub.status.idle":"2024-10-06T09:49:21.667315Z","shell.execute_reply.started":"2024-10-06T09:49:20.559452Z","shell.execute_reply":"2024-10-06T09:49:21.665988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau=bureau.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:49:21.668712Z","iopub.execute_input":"2024-10-06T09:49:21.669096Z","iopub.status.idle":"2024-10-06T09:49:24.211882Z","shell.execute_reply.started":"2024-10-06T09:49:21.669056Z","shell.execute_reply":"2024-10-06T09:49:24.210624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"taxes=pd.concat([\npd.read_csv(dataPath+'csv_files/train/train_tax_registry_a_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4527230A': 'sum','recorddate_4527225D': 'last'}).rename(columns={\"amount_4527230A\": \"amount_taxA\",\"recorddate_4527225D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/train/train_tax_registry_b_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4917619A': 'sum', 'deductiondate_4917603D': 'last'}).rename(columns={\"amount_4917619A\": \"amount_taxA\",\"deductiondate_4917603D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/train/train_tax_registry_c_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'pmtamount_36A': 'sum', 'processingdate_168D': 'last'}).rename(columns={\"pmtamount_36A\": \"amount_taxA\",\"processingdate_168D\":\"recorddate_taxD\"})]\n, axis=0).pipe(Pipeline.filter_cols)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:49:24.213525Z","iopub.execute_input":"2024-10-06T09:49:24.214074Z","iopub.status.idle":"2024-10-06T09:49:41.843074Z","shell.execute_reply.started":"2024-10-06T09:49:24.214018Z","shell.execute_reply":"2024-10-06T09:49:41.841857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_person_1\ndel bureau_a_1\ndel df1\ndel df2\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:49:41.844474Z","iopub.execute_input":"2024-10-06T09:49:41.844840Z","iopub.status.idle":"2024-10-06T09:49:43.078576Z","shell.execute_reply.started":"2024-10-06T09:49:41.844802Z","shell.execute_reply":"2024-10-06T09:49:43.077225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0=train_static_0.set_index(['case_id'])\ntrain_static_cb=train_static_cb.set_index(['case_id'])\ntrain_person1_feats=train_person1_feats.set_index(['case_id'])\nbureau=bureau.set_index(['case_id'])","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:49:43.080267Z","iopub.execute_input":"2024-10-06T09:49:43.080781Z","iopub.status.idle":"2024-10-06T09:49:45.367397Z","shell.execute_reply.started":"2024-10-06T09:49:43.080727Z","shell.execute_reply":"2024-10-06T09:49:45.366056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes= train_basetable.join(train_static_0, how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb, how=\"left\", on=\"case_id\"\n).join(\n    train_person1_feats, how=\"left\", on=\"case_id\"\n).join(\n   bureau, how=\"left\", on=\"case_id\"\n).join(\n   taxes, how=\"left\", on=\"case_id\"\n).pipe(Pipeline.handle_dates)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:49:45.369066Z","iopub.execute_input":"2024-10-06T09:49:45.369493Z","iopub.status.idle":"2024-10-06T09:50:29.281477Z","shell.execute_reply.started":"2024-10-06T09:49:45.369450Z","shell.execute_reply":"2024-10-06T09:50:29.280445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes = bureau_taxes.drop_duplicates('case_id', keep='first')","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:29.282917Z","iopub.execute_input":"2024-10-06T09:50:29.283294Z","iopub.status.idle":"2024-10-06T09:50:30.734926Z","shell.execute_reply.started":"2024-10-06T09:50:29.283257Z","shell.execute_reply":"2024-10-06T09:50:30.733806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=bureau_taxes.drop('dateofbirth_337D',axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:30.736528Z","iopub.execute_input":"2024-10-06T09:50:30.736917Z","iopub.status.idle":"2024-10-06T09:50:31.513765Z","shell.execute_reply.started":"2024-10-06T09:50:30.736878Z","shell.execute_reply":"2024-10-06T09:50:31.512499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.515335Z","iopub.execute_input":"2024-10-06T09:50:31.515737Z","iopub.status.idle":"2024-10-06T09:50:31.523568Z","shell.execute_reply.started":"2024-10-06T09:50:31.515699Z","shell.execute_reply":"2024-10-06T09:50:31.522257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df_train['target']==1).mean()*100","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.525189Z","iopub.execute_input":"2024-10-06T09:50:31.525615Z","iopub.status.idle":"2024-10-06T09:50:31.541788Z","shell.execute_reply.started":"2024-10-06T09:50:31.525569Z","shell.execute_reply":"2024-10-06T09:50:31.540426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.543595Z","iopub.execute_input":"2024-10-06T09:50:31.544113Z","iopub.status.idle":"2024-10-06T09:50:31.553216Z","shell.execute_reply.started":"2024-10-06T09:50:31.544053Z","shell.execute_reply":"2024-10-06T09:50:31.551826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.554822Z","iopub.execute_input":"2024-10-06T09:50:31.555237Z","iopub.status.idle":"2024-10-06T09:50:31.591839Z","shell.execute_reply.started":"2024-10-06T09:50:31.555198Z","shell.execute_reply":"2024-10-06T09:50:31.590728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_static_0\ndel train_static_cb\ndel train_person1_feats\ndel bureau\ndel taxes\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.593126Z","iopub.execute_input":"2024-10-06T09:50:31.593512Z","iopub.status.idle":"2024-10-06T09:50:31.922755Z","shell.execute_reply.started":"2024-10-06T09:50:31.593472Z","shell.execute_reply":"2024-10-06T09:50:31.921518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nfrom sklearn.model_selection import train_test_split\nimport xgboost as xgb\nfrom xgboost import plot_importance\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport pandas as pd\nimport numpy as np\nimport warnings\nfrom xgboost.sklearn import XGBClassifier\nfrom sklearn import metrics","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.924427Z","iopub.execute_input":"2024-10-06T09:50:31.924940Z","iopub.status.idle":"2024-10-06T09:50:31.933856Z","shell.execute_reply.started":"2024-10-06T09:50:31.924886Z","shell.execute_reply":"2024-10-06T09:50:31.932398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=df_train.copy()","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:31.935671Z","iopub.execute_input":"2024-10-06T09:50:31.936237Z","iopub.status.idle":"2024-10-06T09:50:33.518667Z","shell.execute_reply.started":"2024-10-06T09:50:31.936175Z","shell.execute_reply":"2024-10-06T09:50:33.517414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=data.drop(columns=['credtype_322L','disbursementtype_67L','inittransactioncode_186L','lastapprcommoditycat_1041M','lastapprcommoditytypec_5251766M','lastcancelreason_561M','lastrejectcommoditycat_161M','lastrejectcommodtypec_5251769M','lastrejectreason_759M','lastrejectreasonclient_4145040M','lastst_736L','opencred_647L','paytype1st_925L','paytype_783L','previouscontdistrict_112M','twobodfilling_608L','description_5085714M','education_1103M','education_88M','maritalst_385M','maritalst_893M','financialinstitution_591M'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:33.520421Z","iopub.execute_input":"2024-10-06T09:50:33.520835Z","iopub.status.idle":"2024-10-06T09:50:34.120613Z","shell.execute_reply.started":"2024-10-06T09:50:33.520792Z","shell.execute_reply":"2024-10-06T09:50:34.119442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = data.drop(columns=['case_id', 'WEEK_NUM', 'target'])\ny = data['target']","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:34.121845Z","iopub.execute_input":"2024-10-06T09:50:34.122205Z","iopub.status.idle":"2024-10-06T09:50:34.595616Z","shell.execute_reply.started":"2024-10-06T09:50:34.122166Z","shell.execute_reply":"2024-10-06T09:50:34.594261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 划分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:34.597062Z","iopub.execute_input":"2024-10-06T09:50:34.597441Z","iopub.status.idle":"2024-10-06T09:50:36.358500Z","shell.execute_reply.started":"2024-10-06T09:50:34.597402Z","shell.execute_reply":"2024-10-06T09:50:36.357374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(\nbooster='gbtree',\nlearning_rate =0.01,\nn_estimators=50,\nreg_alpha=0.3,\nmax_depth=3,\nmin_child_weight=1,\ngamma=0.1,\nsubsample=0.8,\ncolsample_bytree=0.5,\nearly_stopping=True,\nnthread=4,\nscale_pos_weight=1,\nseed=27) ","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:36.360063Z","iopub.execute_input":"2024-10-06T09:50:36.360481Z","iopub.status.idle":"2024-10-06T09:50:36.367781Z","shell.execute_reply.started":"2024-10-06T09:50:36.360440Z","shell.execute_reply":"2024-10-06T09:50:36.366409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_all=model.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:50:36.369584Z","iopub.execute_input":"2024-10-06T09:50:36.370119Z","iopub.status.idle":"2024-10-06T09:51:07.070246Z","shell.execute_reply.started":"2024-10-06T09:50:36.370061Z","shell.execute_reply":"2024-10-06T09:51:07.069154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model_all.predict(X_test)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:07.072174Z","iopub.execute_input":"2024-10-06T09:51:07.072917Z","iopub.status.idle":"2024-10-06T09:51:13.727400Z","shell.execute_reply.started":"2024-10-06T09:51:07.072850Z","shell.execute_reply":"2024-10-06T09:51:13.726432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = dataPath+'csv_files/test' \nfile_names = os.listdir(folder_path)\n\ntest_bureau_a_1 = pd.DataFrame()\n\nfor file_name in file_names:\n    if file_name.startswith('test_credit_bureau_a_1_') and file_name.endswith('.csv'):\n        file_path = os.path.join(folder_path, file_name)\n        temp_df = pd.read_csv(file_path, usecols=['case_id', 'financialinstitution_591M'])\n        test_bureau_a_1 = pd.concat([test_bureau_a_1, temp_df], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:13.728870Z","iopub.execute_input":"2024-10-06T09:51:13.729851Z","iopub.status.idle":"2024-10-06T09:51:13.820137Z","shell.execute_reply.started":"2024-10-06T09:51:13.729795Z","shell.execute_reply":"2024-10-06T09:51:13.818922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_bureau= pd.concat([test_bureau_a_1,pd.read_csv(dataPath+'csv_files/test/test_credit_bureau_b_1.csv')\\\n.pipe(Pipeline.set_table_dtypes).rename(columns={\"credor_3940957M\": \"financialinstitution_591M\"})[['case_id','financialinstitution_591M']]], axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:13.821584Z","iopub.execute_input":"2024-10-06T09:51:13.821982Z","iopub.status.idle":"2024-10-06T09:51:13.859204Z","shell.execute_reply.started":"2024-10-06T09:51:13.821939Z","shell.execute_reply":"2024-10-06T09:51:13.857857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pd.read_csv(dataPath+'csv_files/test/test_base.csv').pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:13.860791Z","iopub.execute_input":"2024-10-06T09:51:13.861210Z","iopub.status.idle":"2024-10-06T09:51:13.874374Z","shell.execute_reply.started":"2024-10-06T09:51:13.861169Z","shell.execute_reply":"2024-10-06T09:51:13.873234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = test_basetable.assign(\n    month_decision=test_basetable['date_decision'].dt.month,\n    weekday_decision=test_basetable['date_decision'].dt.weekday\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:13.876034Z","iopub.execute_input":"2024-10-06T09:51:13.876699Z","iopub.status.idle":"2024-10-06T09:51:13.886790Z","shell.execute_reply.started":"2024-10-06T09:51:13.876639Z","shell.execute_reply":"2024-10-06T09:51:13.885141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3 =pd.read_csv(dataPath+'csv_files/test/test_static_0_0.csv').pipe(Pipeline.set_table_dtypes)\ndf4 =pd.read_csv(dataPath+'csv_files/test/test_static_0_1.csv').pipe(Pipeline.set_table_dtypes)\ndf5 =pd.read_csv(dataPath+'csv_files/test/test_static_0_2.csv').pipe(Pipeline.set_table_dtypes)\n\ntest_static_0 = pd.concat([df3, df4,df5], axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:13.888671Z","iopub.execute_input":"2024-10-06T09:51:13.889462Z","iopub.status.idle":"2024-10-06T09:51:14.044808Z","shell.execute_reply.started":"2024-10-06T09:51:13.889403Z","shell.execute_reply":"2024-10-06T09:51:14.043459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_static_cb=pd.read_csv(dataPath+'csv_files/test/test_static_cb_0.csv').pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.046454Z","iopub.execute_input":"2024-10-06T09:51:14.046849Z","iopub.status.idle":"2024-10-06T09:51:14.072592Z","shell.execute_reply.started":"2024-10-06T09:51:14.046808Z","shell.execute_reply":"2024-10-06T09:51:14.071393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person_1=pd.read_csv(dataPath+'csv_files/test/test_person_1.csv')\ntest_person_1=test_person_1[test_person_1['num_group1'] == 0].pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.081865Z","iopub.execute_input":"2024-10-06T09:51:14.082360Z","iopub.status.idle":"2024-10-06T09:51:14.107021Z","shell.execute_reply.started":"2024-10-06T09:51:14.082300Z","shell.execute_reply":"2024-10-06T09:51:14.105840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_person1_feats=test_person_1[['case_id','birth_259D','mainoccupationinc_384A']].pipe(Pipeline.set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.108674Z","iopub.execute_input":"2024-10-06T09:51:14.109190Z","iopub.status.idle":"2024-10-06T09:51:14.118924Z","shell.execute_reply.started":"2024-10-06T09:51:14.109134Z","shell.execute_reply":"2024-10-06T09:51:14.117479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_taxes=pd.concat([\npd.read_csv(dataPath+'csv_files/test/test_tax_registry_a_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4527230A': 'sum','recorddate_4527225D': 'last'}).rename(columns={\"amount_4527230A\": \"amount_taxA\",\"recorddate_4527225D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/test/test_tax_registry_b_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'amount_4917619A': 'sum', 'deductiondate_4917603D': 'last'}).rename(columns={\"amount_4917619A\": \"amount_taxA\",\"deductiondate_4917603D\":\"recorddate_taxD\"}),\npd.read_csv(dataPath+'csv_files/test/test_tax_registry_c_1.csv').pipe(Pipeline.set_table_dtypes).groupby('case_id').agg({'pmtamount_36A': 'sum', 'processingdate_168D': 'last'}).rename(columns={\"pmtamount_36A\": \"amount_taxA\",\"processingdate_168D\":\"recorddate_taxD\"})]\n, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.120544Z","iopub.execute_input":"2024-10-06T09:51:14.121018Z","iopub.status.idle":"2024-10-06T09:51:14.181578Z","shell.execute_reply.started":"2024-10-06T09:51:14.120941Z","shell.execute_reply":"2024-10-06T09:51:14.180198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes_test= test_basetable.merge(test_static_0, how=\"left\", on=\"case_id\"\n).merge(\n    test_static_cb, how=\"left\", on=\"case_id\"\n ).merge(\n    test_person1_feats, how=\"left\", on=\"case_id\"\n).merge(\n   test_bureau, how=\"left\", on=\"case_id\"\n ).merge(\n   test_taxes, how=\"left\", on=\"case_id\"\n).pipe(Pipeline.handle_dates)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.183180Z","iopub.execute_input":"2024-10-06T09:51:14.183606Z","iopub.status.idle":"2024-10-06T09:51:14.267281Z","shell.execute_reply.started":"2024-10-06T09:51:14.183563Z","shell.execute_reply":"2024-10-06T09:51:14.266128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bureau_taxes_test = bureau_taxes_test.drop_duplicates('case_id', keep='first')","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.268886Z","iopub.execute_input":"2024-10-06T09:51:14.269312Z","iopub.status.idle":"2024-10-06T09:51:14.278270Z","shell.execute_reply.started":"2024-10-06T09:51:14.269269Z","shell.execute_reply":"2024-10-06T09:51:14.276999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test=bureau_taxes_test.drop('dateofbirth_337D',axis=1)\ndf_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.280216Z","iopub.execute_input":"2024-10-06T09:51:14.280747Z","iopub.status.idle":"2024-10-06T09:51:14.304533Z","shell.execute_reply.started":"2024-10-06T09:51:14.280693Z","shell.execute_reply":"2024-10-06T09:51:14.302679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.set_index('case_id', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.306041Z","iopub.execute_input":"2024-10-06T09:51:14.306493Z","iopub.status.idle":"2024-10-06T09:51:14.314773Z","shell.execute_reply.started":"2024-10-06T09:51:14.306449Z","shell.execute_reply":"2024-10-06T09:51:14.313356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a=list(X_train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.317003Z","iopub.execute_input":"2024-10-06T09:51:14.317531Z","iopub.status.idle":"2024-10-06T09:51:14.326291Z","shell.execute_reply.started":"2024-10-06T09:51:14.317487Z","shell.execute_reply":"2024-10-06T09:51:14.324763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_pred = pd.Series(model_all.predict_proba(df_test[a])[:, 1], index=df_test[a].index)","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.327772Z","iopub.execute_input":"2024-10-06T09:51:14.328183Z","iopub.status.idle":"2024-10-06T09:51:14.365231Z","shell.execute_reply.started":"2024-10-06T09:51:14.328142Z","shell.execute_reply":"2024-10-06T09:51:14.364183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = pd.read_csv(dataPath+\"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.366501Z","iopub.execute_input":"2024-10-06T09:51:14.367772Z","iopub.status.idle":"2024-10-06T09:51:14.385561Z","shell.execute_reply.started":"2024-10-06T09:51:14.367720Z","shell.execute_reply":"2024-10-06T09:51:14.384311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm = df_subm.set_index(\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.387300Z","iopub.execute_input":"2024-10-06T09:51:14.387710Z","iopub.status.idle":"2024-10-06T09:51:14.393965Z","shell.execute_reply.started":"2024-10-06T09:51:14.387670Z","shell.execute_reply":"2024-10-06T09:51:14.392589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm[\"score\"] = xgb_pred","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.395894Z","iopub.execute_input":"2024-10-06T09:51:14.396284Z","iopub.status.idle":"2024-10-06T09:51:14.407498Z","shell.execute_reply.started":"2024-10-06T09:51:14.396246Z","shell.execute_reply":"2024-10-06T09:51:14.406250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.408983Z","iopub.execute_input":"2024-10-06T09:51:14.409421Z","iopub.status.idle":"2024-10-06T09:51:14.426706Z","shell.execute_reply.started":"2024-10-06T09:51:14.409359Z","shell.execute_reply":"2024-10-06T09:51:14.425335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-10-06T09:51:14.428280Z","iopub.execute_input":"2024-10-06T09:51:14.428786Z","iopub.status.idle":"2024-10-06T09:51:14.439688Z","shell.execute_reply.started":"2024-10-06T09:51:14.428731Z","shell.execute_reply":"2024-10-06T09:51:14.438308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}