{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7939528,"sourceType":"datasetVersion","datasetId":4667377}],"dockerImageVersionId":30699,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":94.158852,"end_time":"2024-04-28T17:43:28.574556","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-04-28T17:41:54.415704","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport glob\nimport pickle\nimport pprint\nfrom datetime import datetime\nimport category_encoders as ce\n\nimport warnings\nwarnings.filterwarnings('error', category=DeprecationWarning)","metadata":{"papermill":{"duration":2.913416,"end_time":"2024-04-28T17:42:00.365002","exception":false,"start_time":"2024-04-28T17:41:57.451586","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:47:59.990382Z","iopub.execute_input":"2024-05-12T14:47:59.990829Z","iopub.status.idle":"2024-05-12T14:48:02.231836Z","shell.execute_reply.started":"2024-05-12T14:47:59.990798Z","shell.execute_reply":"2024-05-12T14:48:02.230796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n#     # implement here all desired dtypes for tables\n#     # the following is just an example\n#     for col in df.columns:\n#         # last letter of column name will help you determine the type\n#         if col[-1] in (\"P\", \"A\"):\n#             df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n\n#     return df\ndef reduce_memory_usage(df: pd.DataFrame) -> pd.DataFrame:\n        for col in df.columns:  \n            if df[col].dtype.name in ['object', 'string']:\n                df[col] = df[col].astype(\"string\").astype('category')\n                current_categories = df[col].cat.categories\n                new_categories = current_categories.to_list() + [\"Unknown\"]\n                new_dtype = pd.CategoricalDtype(categories=new_categories, ordered=True)\n                df[col] = df[col].astype(new_dtype)\n            if df[col].dtype in [float]:\n                df[col] = df[col].astype(np.float32)\n        return df","metadata":{"papermill":{"duration":0.021892,"end_time":"2024-04-28T17:42:00.397089","exception":false,"start_time":"2024-04-28T17:42:00.375197","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.233926Z","iopub.execute_input":"2024-05-12T14:48:02.234436Z","iopub.status.idle":"2024-05-12T14:48:02.244412Z","shell.execute_reply.started":"2024-05-12T14:48:02.234402Z","shell.execute_reply":"2024-05-12T14:48:02.243485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loader","metadata":{"papermill":{"duration":0.00917,"end_time":"2024-04-28T17:42:00.415791","exception":false,"start_time":"2024-04-28T17:42:00.406621","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class DataLoader:\n    dataPath = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/'\n\n    cb_a_2_cols = [\n        'case_id',\n        'pmts_overdue_1140A',\n        'pmts_dpd_1073P',\n        'pmts_dpd_303P',\n        'pmts_overdue_1152A',\n    ]\n    \n    person_1_cols = ['case_id',\n                     'empl_employedfrom_271D',\n                     'empl_employedtotal_800L',\n                     'empl_industry_691L', \n                     'birth_259D', \n                     'education_927M', \n                     'incometype_1044T', \n                     'mainoccupationinc_384A', \n                     'registaddr_zipcode_184M',\n                     'num_group1']\n    \n    static_0_cols = ['case_id',\n                     'pctinstlsallpaidlate1d_3546856L',\n                         'pctinstlsallpaidlate4d_3546849L',\n                         'pctinstlsallpaidlate6d_3546844L',\n                         'pctinstlsallpaidlat10d_839L',\n                         'datelastinstal40dpd_247D',\n                         'lastdelinqdate_224D',\n                         'lastrejectdate_50D',\n                         'datelastunpaid_3546854D',\n                         'numinstlswithdpd10_728L',\n                         'avgmaxdpdlast9m_3716943P']\n    \n    static_cb_0_cols = ['case_id',\n                        'riskassesment_940T',\n                         'forquarter_634L',\n                         'formonth_535L',\n                         'days120_123L',\n                         'days180_256L']\n    \n    credit_bureau_b_1 = ['case_id',\n                        'totalamount_881A',\n                         'contractst_516M',]\n    \n    applprev_1 = ['case_id',\n                  'employedfrom_700D',\n                  'actualdpd_943P',\n                  'maxdpdtolerance_577P',\n                  'childnum_21L',\n                  'annuity_853A',\n                  'credacc_credlmt_575A',\n                  'credamount_590A',\n                  'currdebt_94A',\n                  'downpmt_134A',\n                  'mainoccupationinc_437A',\n                  'tenor_203L',\n                  'outstandingdebt_522A',\n                  'familystate_726L']\n    \n    applprev_2 = ['case_id',\n                  'credacc_cards_status_52L']\n    \n    deposit = ['case_id', \n               'openingdate_313D', \n               'amount_416A']\n    \n    debitcard = ['case_id',\n                 'last180dayaveragebalance_704A',\n                 'last180dayturnover_1134A',\n                 'last30dayturnover_651A']\n    \n    other = ['case_id', \n             'amtdepositbalance_4809441A', \n             'amtdepositincoming_4809444A', \n             'amtdepositoutgoing_4809442A', \n             'amtdebitincoming_4809443A', \n             'amtdebitoutgoing_4809440A']\n    \n    tax_registry_a = ['case_id', \n                      'amount_4527230A', \n                      'name_4527232M', \n                      'recorddate_4527225D']\n    \n    tax_registry_b = ['case_id', \n                      'amount_4917619A', \n                      'name_4917606M', \n                      'deductiondate_4917603D']\n    \n    tax_registry_c = ['case_id', \n                      'pmtamount_36A', \n                      'employername_160M', \n                      'processingdate_168D']\n    \n    def __init__(self):\n        self.train_base = pd.read_parquet(self.dataPath + \"train/train_base.parquet\").pipe(reduce_memory_usage)\n        self.test_base = pd.read_parquet(self.dataPath + \"test/test_base.parquet\").pipe(reduce_memory_usage)\n        \n        self.df_target = self.train_base[['target']]\n        \n        self.train_cb_a_2 = pd.concat([pd.read_parquet(file,columns=self.cb_a_2_cols).pipe(reduce_memory_usage) \n                                    for file in glob.glob(self.dataPath + 'train/train_credit_bureau_a_2*.parquet')])\n        \n        self.test_cb_a_2 = pd.concat([pd.read_parquet(file,columns=self.cb_a_2_cols).pipe(reduce_memory_usage) \n                                    for file in glob.glob(self.dataPath + 'test/test_credit_bureau_a_2*.parquet')])\n        \n        self.train_person_1 = pd.read_parquet(self.dataPath + \"train/train_person_1.parquet\",columns = self.person_1_cols).pipe(reduce_memory_usage)\n        \n        self.test_person_1 = pd.read_parquet(self.dataPath + \"test/test_person_1.parquet\",columns = self.person_1_cols).pipe(reduce_memory_usage)\n        \n        self.train_static_0 = pd.concat([\n            pd.read_parquet(self.dataPath + \"train/train_static_0_0.parquet\",columns=self.static_0_cols).pipe(reduce_memory_usage),\n            pd.read_parquet(self.dataPath + \"train/train_static_0_1.parquet\",columns=self.static_0_cols).pipe(reduce_memory_usage),],)\n        self.train_static_cb_0 = pd.read_parquet(self.dataPath + \"train/train_static_cb_0.parquet\",columns=self.static_cb_0_cols).pipe(reduce_memory_usage)\n\n        self.test_static_0 = pd.concat([pd.read_parquet(file,columns=self.static_0_cols).pipe(reduce_memory_usage) \n                                    for file in glob.glob(self.dataPath + 'test/test_static_0*.parquet')])\n        self.test_static_cb_0 = pd.read_parquet(self.dataPath + \"test/test_static_cb_0.parquet\",columns=self.static_cb_0_cols).pipe(reduce_memory_usage)\n        \n        self.train_credit_bureau_b_1 = pd.read_parquet(self.dataPath + \"train/train_credit_bureau_b_1.parquet\",columns = self.credit_bureau_b_1).pipe(reduce_memory_usage)\n        self.test_credit_bureau_b_1 = pd.read_parquet(self.dataPath + \"test/test_credit_bureau_b_1.parquet\",columns = self.credit_bureau_b_1).pipe(reduce_memory_usage)\n        \n        self.train_applprev_1 = pd.concat([\n            pd.read_parquet(self.dataPath + \"train/train_applprev_1_0.parquet\",columns=self.applprev_1).pipe(reduce_memory_usage),\n            pd.read_parquet(self.dataPath + \"train/train_applprev_1_1.parquet\",columns=self.applprev_1).pipe(reduce_memory_usage),],)\n        self.test_applprev_1 = pd.concat([\n            pd.read_parquet(self.dataPath + \"test/test_applprev_1_0.parquet\",columns=self.applprev_1).pipe(reduce_memory_usage),\n            pd.read_parquet(self.dataPath + \"test/test_applprev_1_1.parquet\",columns=self.applprev_1).pipe(reduce_memory_usage),],)\n        \n        self.train_applprev_2 = pd.read_parquet(self.dataPath + \"train/train_applprev_2.parquet\",columns = self.applprev_2).pipe(reduce_memory_usage)\n        self.test_applprev_2 = pd.read_parquet(self.dataPath + \"test/test_applprev_2.parquet\",columns = self.applprev_2).pipe(reduce_memory_usage)\n        \n        self.train_deposit = pd.read_parquet(self.dataPath + \"train/train_deposit_1.parquet\",columns = self.deposit).pipe(reduce_memory_usage)\n        self.test_deposit = pd.read_parquet(self.dataPath + \"test/test_deposit_1.parquet\",columns = self.deposit).pipe(reduce_memory_usage)\n        \n        self.train_debitcard = pd.read_parquet(self.dataPath + \"train/train_debitcard_1.parquet\",columns = self.debitcard).pipe(reduce_memory_usage)\n        self.test_debitcard = pd.read_parquet(self.dataPath + \"test/test_debitcard_1.parquet\",columns = self.debitcard).pipe(reduce_memory_usage)\n        \n        self.train_other = pd.read_parquet(self.dataPath + \"train/train_other_1.parquet\",columns = self.other).pipe(reduce_memory_usage)\n        self.test_other = pd.read_parquet(self.dataPath + \"test/test_other_1.parquet\",columns = self.other).pipe(reduce_memory_usage)\n        \n        self.train_tax_registry_a = pd.read_parquet(self.dataPath + \"train/train_tax_registry_a_1.parquet\",columns = self.tax_registry_a).pipe(reduce_memory_usage)\n        self.test_tax_registry_a = pd.read_parquet(self.dataPath + \"test/test_tax_registry_a_1.parquet\",columns = self.tax_registry_a).pipe(reduce_memory_usage)\n        \n        self.train_tax_registry_b = pd.read_parquet(self.dataPath + \"train/train_tax_registry_b_1.parquet\",columns = self.tax_registry_b).pipe(reduce_memory_usage)\n        self.test_tax_registry_b = pd.read_parquet(self.dataPath + \"test/test_tax_registry_b_1.parquet\",columns = self.tax_registry_b).pipe(reduce_memory_usage)\n        \n        self.train_tax_registry_c = pd.read_parquet(self.dataPath + \"train/train_tax_registry_c_1.parquet\",columns = self.tax_registry_c).pipe(reduce_memory_usage)\n        self.test_tax_registry_c = pd.read_parquet(self.dataPath + \"test/test_tax_registry_c_1.parquet\",columns = self.tax_registry_c).pipe(reduce_memory_usage)","metadata":{"papermill":{"duration":0.030432,"end_time":"2024-04-28T17:42:00.456082","exception":false,"start_time":"2024-04-28T17:42:00.42565","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.245970Z","iopub.execute_input":"2024-05-12T14:48:02.246304Z","iopub.status.idle":"2024-05-12T14:48:02.344193Z","shell.execute_reply.started":"2024-05-12T14:48:02.246274Z","shell.execute_reply":"2024-05-12T14:48:02.343178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{"papermill":{"duration":0.009265,"end_time":"2024-04-28T17:42:00.475306","exception":false,"start_time":"2024-04-28T17:42:00.466041","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler, StandardScaler\nfrom sklearn.cluster import KMeans\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\ndef generate_cluster(train_data, df_to_cluster,n=5):\n    \n    # Normalize the features\n    scaler = MinMaxScaler()\n    train_data_normalized = scaler.fit_transform(train_data)\n    data_for_clustering_normalized = scaler.transform(df_to_cluster)\n\n    # Initialize KMeans with the desired number of clusters\n    n_clusters = n  # You can adjust this number\n    kmeans = KMeans(n_clusters=n_clusters)\n\n    # Fit KMeans clustering to the data\n    kmeans.fit(data_for_clustering_normalized)\n\n    return kmeans.labels_","metadata":{"execution":{"iopub.status.busy":"2024-05-12T14:48:02.346691Z","iopub.execute_input":"2024-05-12T14:48:02.347041Z","iopub.status.idle":"2024-05-12T14:48:02.637128Z","shell.execute_reply.started":"2024-05-12T14:48:02.347008Z","shell.execute_reply":"2024-05-12T14:48:02.636167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_agg_stats(df, groupby_col, columns, method:str):\n        \n    \"\"\"\n    df: data\n    groupby_col: 'case_id'\n    columns: columns you want to agg\n    method: calculation methods like mean, sum, max, ...\n    \"\"\"\n\n    agg_dict = {col: [method] for col in columns}\n    grouped = df.groupby(groupby_col).agg(agg_dict)\n\n    grouped.columns = ['_'.join(col).strip() for col in grouped.columns.values]\n\n    rename_dict = {f'{col}_{method}': f'{col}_{method}' for col in columns}\n    grouped = grouped.rename(columns=rename_dict)\n    return grouped.reset_index()\n\ndef _cb_a_2_features(data):\n    data = data.sort_values('case_id')\n    df_features = calculate_agg_stats(data,'case_id', [col for col in data.columns if col[-1] in ['L', 'A', 'P']],'mean')\n    return df_features\n\ndef _general_features(df_features):\n    df_features = df_features.drop(columns=['WEEK_NUM', \"date_decision\",\"MONTH\"])\n    return df_features","metadata":{"papermill":{"duration":0.022454,"end_time":"2024-04-28T17:42:00.507242","exception":false,"start_time":"2024-04-28T17:42:00.484788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.638341Z","iopub.execute_input":"2024-05-12T14:48:02.638728Z","iopub.status.idle":"2024-05-12T14:48:02.647004Z","shell.execute_reply.started":"2024-05-12T14:48:02.638695Z","shell.execute_reply":"2024-05-12T14:48:02.646091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_missing_columns(dataframe):\n\n    missing_percentage = dataframe.isna().mean() * 100\n\n    # Identify columns to drop: Ends with 'D' and more than 50% missing\n    columns_to_drop = [col for col in dataframe.columns \n                       if missing_percentage[col] > 50]\n\n    # Drop the identified columns\n    dataframe_dropped = dataframe.drop(columns=columns_to_drop)\n\n    return dataframe_dropped,columns_to_drop\n\ndef calculate_days_to_now(data):\n\n    today = pd.Timestamp.now().normalize()\n    date_columns = [col for col in data.columns if col.endswith('D')]\n\n    for col in date_columns:\n        try:\n            data[col] = pd.to_datetime(data[col])\n            data[col] = (today - data[col]).dt.days\n        except Exception as e:\n            raise ValueError(f\"Column '{col}' could not be converted to datetime: {e}\")\n    return data\n\ndef drop_unique_columns(data):\n    \"\"\"\n    Drop columns that contain only unique values.\n    \"\"\"\n    columns_to_drop = [col for col in data.columns if data[col].nunique() <= 1]\n    data.drop(columns=columns_to_drop, inplace=True)\n    return data, columns_to_drop\n    \ndef weight_of_evidence(df_train, df_test,train_base,test_base):\n    df_train = train_base.drop(columns=['WEEK_NUM', \"date_decision\",\"MONTH\",'target']).merge(df_train,on='case_id',how='left')\n    df_test = test_base.drop(columns=['WEEK_NUM', \"date_decision\",\"MONTH\"]).merge(df_test,on='case_id',how='left')\n    cat_cols = list(df_train.select_dtypes(include=['category','object']).columns)\n    encoder = ce.WOEEncoder(cols=cat_cols,drop_invariant=False)\n    df_target = train_base['target']\n    encoder.fit(df_train, df_target)\n    train_features = encoder.transform(df_train)\n    test_features = encoder.transform(df_test)\n    return train_features,test_features","metadata":{"papermill":{"duration":0.025245,"end_time":"2024-04-28T17:42:00.541993","exception":false,"start_time":"2024-04-28T17:42:00.516748","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.648281Z","iopub.execute_input":"2024-05-12T14:48:02.648643Z","iopub.status.idle":"2024-05-12T14:48:02.661282Z","shell.execute_reply.started":"2024-05-12T14:48:02.648611Z","shell.execute_reply":"2024-05-12T14:48:02.660417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_person_1(test_person_1, train_person_1, train_base):\n    ##no duplicated caseid if only keep num_group = 0\n    #0: applicant\n    test_person_1 = test_person_1[test_person_1['num_group1']==0]\n    train_person_1 = train_person_1[train_person_1['num_group1']==0]\n    \n    #define useful columns\n    useful_cols = ['case_id','empl_employedfrom_271D','empl_employedtotal_800L','empl_industry_691L', 'birth_259D', 'education_927M', 'incometype_1044T', 'mainoccupationinc_384A', 'registaddr_zipcode_184M']\n    \n    test_person_1 = test_person_1[useful_cols]\n    train_person_1 = train_person_1[useful_cols]\n    \n    #create new column: employment (if the applicant is employed then return 1, else return 0)\n    train_person_1['employment'] = train_person_1[['empl_employedfrom_271D','empl_employedtotal_800L','empl_industry_691L']].notnull().any(axis=1).astype(int)\n    train_person_1 = train_person_1.drop(columns = ['empl_employedfrom_271D','empl_employedtotal_800L','empl_industry_691L'])\n    test_person_1['employment'] = test_person_1[['empl_employedfrom_271D','empl_employedtotal_800L','empl_industry_691L']].notnull().any(axis=1).astype(int)\n    test_person_1 = test_person_1.drop(columns = ['empl_employedfrom_271D','empl_employedtotal_800L','empl_industry_691L'])\n    \n    #drop rows with na\n    #test_person_1 = test_person_1.dropna(how='any')\n    train_person_1 = train_person_1.dropna(how='any')\n    \n    #create new column : Age (return age of the applicant)\n    train_person_1['birth_259D'] = pd.to_datetime(train_person_1['birth_259D'])\n    current_date = datetime.now()\n    train_person_1['Age'] = (current_date - train_person_1['birth_259D']) // pd.Timedelta(days=365.25)\n    train_person_1['Age'] = train_person_1['Age'].astype(int)\n    train_person_1['Age_Cluster'] = generate_cluster(train_person_1[['Age']], train_person_1[['Age']])\n    #train_person_1['registaddr_zipcode_184M'] = train_person_1['registaddr_zipcode_184M'].astype(int)\n    #train_person_1['registaddr_zipcode_184M_Cluster'] = generate_cluster(train_person_1[['registaddr_zipcode_184M']], train_person_1[['registaddr_zipcode_184M']],10)\n    test_person_1['birth_259D'] = pd.to_datetime(test_person_1['birth_259D'])\n    test_person_1['Age'] = (current_date - test_person_1['birth_259D']) // pd.Timedelta(days=365.25)\n    test_person_1['Age'] = test_person_1['Age'].astype(int)\n    test_person_1['Age_Cluster'] = generate_cluster(train_person_1[['Age']],test_person_1[['Age']])\n    #test_person_1['registaddr_zipcode_184M'] = test_person_1['registaddr_zipcode_184M'].astype(int)\n    #test_person_1['registaddr_zipcode_184M_Cluster'] = generate_cluster(train_person_1[['registaddr_zipcode_184M']],test_person_1[['registaddr_zipcode_184M']],10)\n    train_person_1.drop(columns=['Age'],inplace=True)\n    test_person_1.drop(columns=['Age'],inplace=True)\n    \n    train_person_1 = train_person_1.drop(columns=['birth_259D'])\n    test_person_1 = test_person_1.drop(columns=['birth_259D'])\n    \n    #target encoding for address related var\n    \n    #merge with targes\n    train_person_1 = train_person_1.merge(train_base[['case_id','target']], how='left', on='case_id')\n    mean_dr = len(train_base[train_base['target'] == 1]) / len(train_base)\n    del train_base\n    \n    def get_drdist(colname):\n        #function for calculate rate of default under different levels for one specified column\n        dr = train_person_1[train_person_1['target'] == 1][[colname,'target']].groupby(colname).size() / train_person_1[[colname,'target']].groupby(colname).size()\n        dist = train_person_1[[colname,'target']].groupby(colname).size()\n        return dr.fillna(0), dist   #return a pd.Series\n    ##################\n    ##create mapping relations\n    ###################\n    \n    #mapping relations will be stored in a dictionary\n    #mapping relations are calculated from train data\n    mapping_dic = dict()\n    #variables waiting for ordinal encoding\n    ord_enc_var = ['education_927M','incometype_1044T']\n    for col in ord_enc_var:\n        dr,_ = get_drdist(col)\n        rank = dict(dr.rank()-1)\n        mapping_dic.update({col:rank})\n    \n\n    #variables related to address (zipcode)\n    #encoding schemes: labeled as 1 if rate of defaulted loan cases under different level > threhold, otherwise labeled as 0\n    address_related_var_person1 = ['registaddr_zipcode_184M']\n    for col in address_related_var_person1:\n        dr, dist = get_drdist(col)\n        drdist_df = pd.concat([dr, dist], axis=1)\n        drdist_df['enc'] = 100 * drdist_df[0]\n        drdist_df.loc[drdist_df[1] <= 50, 'enc'] = mean_dr * 100\n        enc_target = drdist_df['enc'].to_dict()\n        mapping_dic.update({col:enc_target})\n    \n    ##################\n    ##Encoding\n    ###################\n    for col in ord_enc_var + address_related_var_person1:\n        mapping = mapping_dic[col]\n        train_person_1[col] = train_person_1[col].map(mapping)\n        test_person_1[col] = test_person_1[col].map(mapping)\n    \n    \n    return test_person_1, train_person_1","metadata":{"papermill":{"duration":0.116442,"end_time":"2024-04-28T17:42:00.668399","exception":false,"start_time":"2024-04-28T17:42:00.551957","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.662487Z","iopub.execute_input":"2024-05-12T14:48:02.662762Z","iopub.status.idle":"2024-05-12T14:48:02.683074Z","shell.execute_reply.started":"2024-05-12T14:48:02.662739Z","shell.execute_reply":"2024-05-12T14:48:02.682224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#====================== cb preprocessing functions ===================\ndef pre_base(train_base):\n    train_base['date_decision'] = pd.to_datetime(train_base['date_decision'])\n    return train_base\n\n# Append Decision Dates\ndef append_decision(df,train_base):\n    df = df.merge(train_base[['case_id', 'date_decision']], on='case_id', how='left')\n    return df\n\n# transform Type D to dates\ndef proc_D(df):\n    for column in df.columns:\n      # Check if column name ends with \"D\"\n      if column.endswith(\"D\"):\n          # Convert to datetime\n          df[column] = pd.to_datetime(df[column])\n    return df\n\n# Transform, Type D to difference between decision day and itself\ndef proc_D_recency(df):\n    for column in df.columns:\n      # Check if column name ends with \"D\"\n      if column.endswith(\"D\"):\n          # Convert to recency\n          df[column] = df[\"date_decision\"] - df[column]\n          # Convert to numeric\n          df[column] = df[column].astype(int) // 10**9  # Convert nanoseconds to seconds\n    return df\n\n# transform Type T to string\ndef proc_T(df):\n    for column in df.columns:\n      # Check if column name ends with \"T\"\n      if column.endswith(\"T\"):\n          # Convert to string\n          df[column] = df[column].astype(str)\n    return df\n\n# Append Target Column\ndef append_target(df, train_base):\n    df = df.merge(train_base[['case_id', 'target']], on='case_id', how='left')\n    return df\n\n# WOE for M, L, T (after converting T to string)\ndef proc_cat_woe(df):\n    # Filter columns that meet the criteria\n    columns_to_encode = [col for col in df.columns if col.endswith(('M', 'L', 'T')) and df[col].dtype == 'category']\n\n    # Create a WeightOfEvidenceEncoder object\n    woe_encoder = ce.WOEEncoder()\n\n    # Fit and transform the selected columns\n    df[columns_to_encode] = woe_encoder.fit_transform(df[columns_to_encode], df['target'])\n\n    return df, woe_encoder\n\n# WOE for M, L, T (after converting T to string) for TEST\ndef proc_cat_woe_test(df,woe_encoder):\n    # Filter columns that meet the criteria\n    columns_to_encode = [col for col in df.columns if col.endswith(('M', 'L', 'T')) and df[col].dtype == 'category']\n\n    # Transform the selected columns\n    df[columns_to_encode] = woe_encoder.transform(df[columns_to_encode])\n\n    return df","metadata":{"papermill":{"duration":0.025545,"end_time":"2024-04-28T17:42:00.70341","exception":false,"start_time":"2024-04-28T17:42:00.677865","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.684355Z","iopub.execute_input":"2024-05-12T14:48:02.684680Z","iopub.status.idle":"2024-05-12T14:48:02.698081Z","shell.execute_reply.started":"2024-05-12T14:48:02.684656Z","shell.execute_reply":"2024-05-12T14:48:02.697317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_train(df, train_base):\n    \"\"\"preprocessing train dfs including data type transoformation and categorical encoding, takes train and base, returns processed train and the cat encoder\"\"\"\n    # process base\n    train_base = pre_base(train_base)\n    # append date decision\n    df = append_decision(df,train_base)\n    # transform columns ending with D\n    df = proc_D(df)\n    # transform columns ending with D to recency relative to decision date\n    df = proc_D_recency(df)\n\n    # convert columns ending with T to string\n    df = proc_T(df)\n    # append target\n    df = append_target(df, train_base)\n    # WOE encoding for categorical columns ending with M,L,T\n    df, woe_encoder = proc_cat_woe(df)\n\n    return df, woe_encoder","metadata":{"papermill":{"duration":0.019833,"end_time":"2024-04-28T17:42:00.73284","exception":false,"start_time":"2024-04-28T17:42:00.713007","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.699082Z","iopub.execute_input":"2024-05-12T14:48:02.699313Z","iopub.status.idle":"2024-05-12T14:48:02.710980Z","shell.execute_reply.started":"2024-05-12T14:48:02.699293Z","shell.execute_reply":"2024-05-12T14:48:02.710170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_test(df, test_base, woe_encoder):\n    \"\"\"preprocessing test dfs including data type transoformation and categorical encoding, takes test and base, returns processed test \"\"\"\n    # process base\n    test_base = pre_base(test_base)\n    # append date decision\n    df = append_decision(df,test_base)\n    # transform columns ending with D\n    df = proc_D(df)\n    # transform columns ending with D to recency relative to decision date\n    df = proc_D_recency(df)\n\n    # convert columns ending with T to string\n    df = proc_T(df)\n    # WOE encoding for categorical columns ending with M,L,T\n    df = proc_cat_woe_test(df,woe_encoder)\n\n    return df","metadata":{"papermill":{"duration":0.01961,"end_time":"2024-04-28T17:42:00.762039","exception":false,"start_time":"2024-04-28T17:42:00.742429","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.715227Z","iopub.execute_input":"2024-05-12T14:48:02.715497Z","iopub.status.idle":"2024-05-12T14:48:02.720945Z","shell.execute_reply.started":"2024-05-12T14:48:02.715475Z","shell.execute_reply":"2024-05-12T14:48:02.720214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_by_id(df,method):\n    agg=calculate_agg_stats(df,\n                            'case_id', \n                           [col for col in df.columns if col not in [\"case_id\", \"target\", \"date_decision\", \"num_group1\", \"num_group2\"]],\n                           method)\n    return agg","metadata":{"papermill":{"duration":0.019338,"end_time":"2024-04-28T17:42:00.791137","exception":false,"start_time":"2024-04-28T17:42:00.771799","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.721975Z","iopub.execute_input":"2024-05-12T14:48:02.722239Z","iopub.status.idle":"2024-05-12T14:48:02.733694Z","shell.execute_reply.started":"2024-05-12T14:48:02.722218Z","shell.execute_reply":"2024-05-12T14:48:02.732904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def proprocess_prev(applprev_1, base):\n    import pandas as pd\n    from datetime import datetime\n    \n    # Create new numerical variable 'employyear' from 'employedfrom_700D'\n    applprev_1['employedfrom_700D'] = pd.to_datetime(applprev_1['employedfrom_700D'])\n    today = datetime.now() \n    applprev_1['employyear'] = (today - applprev_1['employedfrom_700D']).dt.days // 365\n\n    # Group by 'case_id' and aggregate the numerical columns\n    applprev_1 = applprev_1.groupby('case_id').agg({\n        'actualdpd_943P': 'max',\n        'maxdpdtolerance_577P': 'max',\n        'childnum_21L': 'max',\n        'annuity_853A': 'mean',\n        'credacc_credlmt_575A': 'mean',\n        'credamount_590A': 'mean',\n        'currdebt_94A': 'mean',\n        'downpmt_134A': 'mean',\n        'mainoccupationinc_437A': 'mean',\n        'tenor_203L': 'mean',\n        'outstandingdebt_522A': 'mean',\n        'employyear': 'mean'\n    }).reset_index()\n    \n    # Merge with base\n    applprev = pd.merge(base[['case_id']], applprev_1, on='case_id', how='left')\n    \n    return applprev\n\n\ndef preprocess_dep(deposit, base):\n    # Sort the DataFrame by 'case_id' and 'openingdate'\n    deposit = deposit.sort_values(by=['amount_416A'],ascending=True)\n    deposit = deposit.sort_values(by=['case_id', 'openingdate_313D'])\n    \n    # Filter rows where 'mount' is larger than 0\n    positive_mount_df = deposit[deposit['amount_416A'] > 0]\n\n    # Group by 'case_id' and keep the last row for each group\n    deposit = positive_mount_df.groupby('case_id').tail(1)\n    deposit = deposit[['case_id', 'amount_416A']]\n\n    # Merge with base\n    deposit = pd.merge(base[['case_id']], deposit, on='case_id', how='left')\n    \n    return deposit\n\n\ndef preprocess_deb(debitcard, base):\n    # Calculate the average value for each 'case_id'\n    debitcard = debitcard.groupby('case_id').mean().reset_index()\n\n    #merge with base\n    debitcard = pd.merge(base[['case_id']], debitcard, on='case_id', how='left')\n    \n    return debitcard\n\n\ndef preprocess_other(other, base):\n    #merge with base\n    other = pd.merge(base[['case_id']], other, on='case_id', how='left')\n    \n    return other\n\n\ndef proprocess_tax(tax_registry_a, tax_registry_b, tax_registry_c, base):\n        # Define a dictionary mapping current column names to new column names\n    column_name_mapping_a = {\n        'case_id': 'case_id',\n        'amount_4527230A': 'pmtamount',\n        'name_4527232M': 'employername',\n        'recorddate_4527225D': 'processingdate',\n    }\n    \n    column_name_mapping_b = {\n        'case_id': 'case_id',\n        'amount_4917619A': 'pmtamount',\n        'name_4917606M': 'employername',\n        'deductiondate_4917603D': 'processingdate',\n    }\n    \n    column_name_mapping_c = {\n        'case_id': 'case_id',\n        'pmtamount_36A': 'pmtamount',\n        'employername_160M': 'employername',\n        'processingdate_168D': 'processingdate',\n    }\n    \n    # Change column names using the dictionary mapping\n    tax_registry_a = tax_registry_a.rename(columns=column_name_mapping_a)\n    tax_registry_b = tax_registry_b.rename(columns=column_name_mapping_b)\n    tax_registry_c = tax_registry_c.rename(columns=column_name_mapping_c)\n    \n    # Combine the three DataFrames into one\n    tax_registry = pd.concat([tax_registry_c, tax_registry_b, tax_registry_a])\n\n    # Calculate the average of 'pmtamount' for each 'case_id'\n    tax_registry = tax_registry.groupby('case_id')['pmtamount'].mean().reset_index()\n        \n    #merge with targes\n    tax_registry = pd.merge(base[['case_id']], tax_registry, on='case_id', how='left')\n        \n    return tax_registry\n","metadata":{"execution":{"iopub.status.busy":"2024-05-12T14:48:02.735658Z","iopub.execute_input":"2024-05-12T14:48:02.735950Z","iopub.status.idle":"2024-05-12T14:48:02.751001Z","shell.execute_reply.started":"2024-05-12T14:48:02.735926Z","shell.execute_reply":"2024-05-12T14:48:02.750041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = DataLoader()","metadata":{"papermill":{"duration":33.384998,"end_time":"2024-04-28T17:42:34.185576","exception":false,"start_time":"2024-04-28T17:42:00.800578","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:02.752138Z","iopub.execute_input":"2024-05-12T14:48:02.752462Z","iopub.status.idle":"2024-05-12T14:48:39.052156Z","shell.execute_reply.started":"2024-05-12T14:48:02.752432Z","shell.execute_reply":"2024-05-12T14:48:39.051286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cb_b1 data\ntrain_credit_bureau_b_1, woe_encoder_cbb1 = preprocess_train(data.train_credit_bureau_b_1,data.train_base)\ntest_credit_bureau_b_1 = preprocess_test(data.test_credit_bureau_b_1, data.test_base, woe_encoder_cbb1)\n\n# aggregate cb_b1 data\ntrain_credit_bureau_b_1 = aggregate_by_id(train_credit_bureau_b_1,'mean')\ntest_credit_bureau_b_1 = aggregate_by_id(test_credit_bureau_b_1,'mean')","metadata":{"papermill":{"duration":0.26283,"end_time":"2024-04-28T17:42:34.457966","exception":false,"start_time":"2024-04-28T17:42:34.195136","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:39.053352Z","iopub.execute_input":"2024-05-12T14:48:39.053649Z","iopub.status.idle":"2024-05-12T14:48:39.276213Z","shell.execute_reply.started":"2024-05-12T14:48:39.053623Z","shell.execute_reply":"2024-05-12T14:48:39.275160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_applprev = proprocess_prev(data.train_applprev_1, data.train_base)\ntest_applprev = proprocess_prev(data.test_applprev_1, data.test_base)\n\ntrain_deposit = preprocess_dep(data.train_deposit, data.train_base)\ntest_deposit = preprocess_dep(data.test_deposit, data.test_base)\n\ntrain_debitcard = preprocess_deb(data.train_debitcard, data.train_base)\ntest_debitcard = preprocess_deb(data.test_debitcard, data.test_base)\n\ntrain_other = preprocess_other(data.train_other, data.train_base)\ntest_other = preprocess_other(data.test_other, data.test_base)\n\ntrain_tax_registry = proprocess_tax(data.train_tax_registry_a, data.train_tax_registry_b, data.train_tax_registry_c, data.train_base)\ntest_tax_registry = proprocess_tax(data.test_tax_registry_a, data.test_tax_registry_b, data.test_tax_registry_c, data.test_base)","metadata":{"execution":{"iopub.status.busy":"2024-05-12T14:48:39.277516Z","iopub.execute_input":"2024-05-12T14:48:39.277935Z","iopub.status.idle":"2024-05-12T14:48:43.151902Z","shell.execute_reply.started":"2024-05-12T14:48:39.277903Z","shell.execute_reply":"2024-05-12T14:48:43.150942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev = proprocess_prev(data.test_applprev_1, data.test_base)\ntest_applprev","metadata":{"execution":{"iopub.status.busy":"2024-05-12T14:48:43.153272Z","iopub.execute_input":"2024-05-12T14:48:43.153661Z","iopub.status.idle":"2024-05-12T14:48:43.190161Z","shell.execute_reply.started":"2024-05-12T14:48:43.153625Z","shell.execute_reply":"2024-05-12T14:48:43.189309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## person data 1\npre_test_person_1, pre_train_person_1 = preprocess_person_1(data.test_person_1,data.train_person_1,data.train_base)","metadata":{"papermill":{"duration":0.596471,"end_time":"2024-04-28T17:42:35.064082","exception":false,"start_time":"2024-04-28T17:42:34.467611","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:43.191340Z","iopub.execute_input":"2024-05-12T14:48:43.191656Z","iopub.status.idle":"2024-05-12T14:48:46.720193Z","shell.execute_reply.started":"2024-05-12T14:48:43.191630Z","shell.execute_reply":"2024-05-12T14:48:46.719338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# static\ndata.train_static_0 = data.train_static_0.pipe(calculate_days_to_now)\ndata.test_static_0 = data.test_static_0.pipe(calculate_days_to_now)\ntrain_static_0_features,test_static_0_features = weight_of_evidence(data.train_static_0, data.test_static_0,data.train_base,data.test_base)\n\n# static_cb\ndata.train_static_cb_0 = data.train_static_cb_0.pipe(calculate_days_to_now)\ndata.test_static_cb_0 = data.test_static_cb_0.pipe(calculate_days_to_now)\ntrain_static_cb_0_features,test_static_cb_0_features = weight_of_evidence(data.train_static_cb_0, data.test_static_cb_0,data.train_base,data.test_base)","metadata":{"papermill":{"duration":1.868749,"end_time":"2024-04-28T17:42:36.942571","exception":false,"start_time":"2024-04-28T17:42:35.073822","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:46.721588Z","iopub.execute_input":"2024-05-12T14:48:46.722257Z","iopub.status.idle":"2024-05-12T14:48:48.584277Z","shell.execute_reply.started":"2024-05-12T14:48:46.722221Z","shell.execute_reply":"2024-05-12T14:48:48.583022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features = _general_features(data.train_base).merge(\n        _cb_a_2_features(data.train_cb_a_2),\n        on=[\"case_id\"],\n        how=\"left\",\n    ).merge(\n    pre_train_person_1.drop(columns='target'),on='case_id',how='left'\n).merge(\n    train_static_0_features,on='case_id',how='left'\n).merge(\n    train_static_cb_0_features,on='case_id',how='left'\n).merge(\n    train_credit_bureau_b_1,on='case_id',how='left'\n).merge(\n    train_applprev,on='case_id',how='left'\n).merge(\n    train_deposit,on='case_id',how='left'\n).merge(\n    train_debitcard,on='case_id',how='left'\n).merge(\n    train_other,on='case_id',how='left'\n).merge(\n    train_tax_registry,on='case_id',how='left'\n).pipe(reduce_memory_usage)\n\ndf_target = data.df_target","metadata":{"papermill":{"duration":32.119432,"end_time":"2024-04-28T17:43:09.07159","exception":false,"start_time":"2024-04-28T17:42:36.952158","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:48:48.585894Z","iopub.execute_input":"2024-05-12T14:48:48.586194Z","iopub.status.idle":"2024-05-12T14:49:20.663479Z","shell.execute_reply.started":"2024-05-12T14:48:48.586169Z","shell.execute_reply":"2024-05-12T14:49:20.662662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop target from train_features\ntrain_features = train_features.drop(columns='target')","metadata":{"papermill":{"duration":0.110635,"end_time":"2024-04-28T17:43:09.192221","exception":false,"start_time":"2024-04-28T17:43:09.081586","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:20.664626Z","iopub.execute_input":"2024-05-12T14:49:20.664943Z","iopub.status.idle":"2024-05-12T14:49:20.810147Z","shell.execute_reply.started":"2024-05-12T14:49:20.664916Z","shell.execute_reply":"2024-05-12T14:49:20.809137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features,missing_cols_dropped = drop_missing_columns(train_features)\ntrain_features,unique_cols_dropped = drop_unique_columns(train_features)","metadata":{"papermill":{"duration":0.590003,"end_time":"2024-04-28T17:43:09.791994","exception":false,"start_time":"2024-04-28T17:43:09.201991","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:20.811622Z","iopub.execute_input":"2024-05-12T14:49:20.812004Z","iopub.status.idle":"2024-05-12T14:49:21.738979Z","shell.execute_reply.started":"2024-05-12T14:49:20.811971Z","shell.execute_reply":"2024-05-12T14:49:21.738181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for col in train_features.select_dtypes(['category']).columns:\n#     train_features[col] = train_features[col].cat.add_categories(-1)\n\n# train_features.fillna(-1,inplace=True)","metadata":{"papermill":{"duration":0.018754,"end_time":"2024-04-28T17:43:09.820434","exception":false,"start_time":"2024-04-28T17:43:09.80168","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:21.740064Z","iopub.execute_input":"2024-05-12T14:49:21.740348Z","iopub.status.idle":"2024-05-12T14:49:21.744295Z","shell.execute_reply.started":"2024-05-12T14:49:21.740323Z","shell.execute_reply":"2024-05-12T14:49:21.743408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features.isnull().mean() * 100","metadata":{"papermill":{"duration":0.024066,"end_time":"2024-04-28T17:43:09.855348","exception":false,"start_time":"2024-04-28T17:43:09.831282","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:21.745257Z","iopub.execute_input":"2024-05-12T14:49:21.745561Z","iopub.status.idle":"2024-05-12T14:49:21.811459Z","shell.execute_reply.started":"2024-05-12T14:49:21.745532Z","shell.execute_reply":"2024-05-12T14:49:21.810575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train a model","metadata":{"papermill":{"duration":0.009519,"end_time":"2024-04-28T17:43:09.874895","exception":false,"start_time":"2024-04-28T17:43:09.865376","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom bayes_opt import BayesianOptimization\n# from sklearn.model_selection import StratifiedKFold\n# from skopt import BayesSearchCV\n# from skopt.callbacks import DeadlineStopper, DeltaYStopper\n# from skopt.space import Real, Categorical, Integer","metadata":{"papermill":{"duration":1.111936,"end_time":"2024-04-28T17:43:10.997078","exception":false,"start_time":"2024-04-28T17:43:09.885142","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:21.812757Z","iopub.execute_input":"2024-05-12T14:49:21.813048Z","iopub.status.idle":"2024-05-12T14:49:24.803531Z","shell.execute_reply.started":"2024-05-12T14:49:21.813024Z","shell.execute_reply":"2024-05-12T14:49:24.802664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def bayes_opt_lgb(X, y, init_round=15, opt_round=25, n_folds=10, random_seed=42, n_estimators=5000, output_process=False):\n#     train_data = lgb.Dataset(data=X, label=y)\n\n#     def lgb_eval(num_leaves, feature_fraction, bagging_fraction,learning_rate, max_depth, lambda_l1, lambda_l2, subsample,subsample_freq,\n#                  min_child_weight,colsample_bytree,min_gain_to_split,min_data_in_leaf):\n#         params = {'application':'binary', 'early_stopping_round':100,'verbose':-1,'feature_pre_filter':False,'boosting':'gbdt','device':'gpu', 'scale_pos_weight':31} #add bias-correction for imbalanced data\n#         params[\"num_leaves\"] = int(round(num_leaves))\n#         params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n#         params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n#         params['learning_rate'] = learning_rate\n#         params['max_depth'] = int(round(max_depth))\n#         params['lambda_l1'] = max(lambda_l1, 0)\n#         params['lambda_l2'] = max(lambda_l2, 0)\n#         params['subsample'] = subsample\n#         params['subsample_freq'] = int(round(subsample_freq))\n#         params['min_child_weight'] = min_child_weight\n#         params['colsample_bytree'] = colsample_bytree\n#         params['min_gain_to_split'] = min_gain_to_split\n#         params['min_data_in_leaf'] = int(round(min_data_in_leaf))\n#         cv_result = lgb.cv(params, train_data,num_boost_round=n_estimators, nfold=n_folds, seed=random_seed, stratified=True, metrics=['auc'])\n#         return max(cv_result['valid auc-mean'])\n\n#     lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (20, 3000),\n#                                             'feature_fraction': (0.1, 0.9),\n#                                             'bagging_fraction': (0.8, 1),\n#                                             \"learning_rate\": (0.01, 1.0),\n#                                             'max_depth': (-1, 256),\n#                                             'lambda_l1': (0, 100),\n#                                             'lambda_l2': (0, 100),\n#                                             'min_gain_to_split': (0.001, 15),\n#                                             \"subsample\": (0.01, 1.0),\n#                                             'subsample_freq': (1, 10),\n#                                             'colsample_bytree': (0.01, 1.0),\n#                                             \"min_data_in_leaf\": (200, 10000),\n#                                             'min_child_weight': (5, 50)}, random_state=0)\n#     # optimize\n#     lgbBO.maximize(init_points=init_round, n_iter=opt_round)\n    \n#     if output_process==True: \n#         lgbBO.points_to_csv(\"/kaggle/working/bayes_opt_result.csv\")\n\n#     return lgbBO\n# opt_params = bayes_opt_lgb(train_features, df_target, init_round=5, opt_round=10, n_folds=5, random_seed=42, n_estimators=100)","metadata":{"papermill":{"duration":0.019908,"end_time":"2024-04-28T17:43:11.0268","exception":false,"start_time":"2024-04-28T17:43:11.006892","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.804985Z","iopub.execute_input":"2024-05-12T14:49:24.805479Z","iopub.status.idle":"2024-05-12T14:49:24.811689Z","shell.execute_reply.started":"2024-05-12T14:49:24.805443Z","shell.execute_reply":"2024-05-12T14:49:24.810814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#opt_params.max['params']","metadata":{"execution":{"iopub.status.busy":"2024-05-12T14:49:24.812967Z","iopub.execute_input":"2024-05-12T14:49:24.813297Z","iopub.status.idle":"2024-05-12T14:49:24.824111Z","shell.execute_reply.started":"2024-05-12T14:49:24.813266Z","shell.execute_reply":"2024-05-12T14:49:24.823216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# opt_params.max['params']","metadata":{"papermill":{"duration":0.019195,"end_time":"2024-04-28T17:43:11.055831","exception":false,"start_time":"2024-04-28T17:43:11.036636","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.825216Z","iopub.execute_input":"2024-05-12T14:49:24.825737Z","iopub.status.idle":"2024-05-12T14:49:24.834038Z","shell.execute_reply.started":"2024-05-12T14:49:24.825707Z","shell.execute_reply":"2024-05-12T14:49:24.833264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**不要用这个下面的cell训练模型**","metadata":{"papermill":{"duration":0.009311,"end_time":"2024-04-28T17:43:11.075076","exception":false,"start_time":"2024-04-28T17:43:11.065765","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\n\n# clf = lgb.LGBMClassifier(boosting_type='gbdt',\n#                         objective='binary',\n#                         metric='roc_auc',\n#                          missing=-1,\n#                         n_jobs=1, \n#                         verbose=-1,\n#                         random_state=0)\n\n# search_spaces = {\n#     'learning_rate': Real(0.01, 1.0, 'log-uniform'),    \n#     'n_estimators': Integer(30, 200),              \n#     'num_leaves': Integer(20, 3000),\n#     'max_depth': Integer(-1, 256),\n#     'subsample': Real(0.01, 1.0, 'uniform'),\n#     'subsample_freq': Integer(1, 10),\n#     'colsample_bytree': Real(0.01, 1.0, 'uniform'),\n#     'reg_lambda': Real(1e-9, 100.0, 'log-uniform'),\n#     'reg_alpha': Real(1e-9, 100.0, 'log-uniform'),\n#     \"min_gain_to_split\": Real(0.001, 15, 'log-uniform'),\n#     \"bagging_fraction\": Real(0.8, 1, 'uniform'),\n#     \"feature_fraction\": Real(0.2, 0.9,'uniform'),\n#     \"min_data_in_leaf\": Integer(200, 10000),\n    \n#    }\n\n# optimizer = BayesSearchCV(estimator=clf,                                    \n#                     search_spaces=search_spaces,                      \n#                     scoring='roc_auc',                           \n#                     cv=skf,                                                                              \n#                     n_iter=20,                                        # max number of trials\n#                     n_points=3,                                       # number of hyperparameter sets evaluated at the same time\n#                     n_jobs=-1,                                        # number of jobs\n#                     return_train_score=False,\n#                     refit=False,                                      \n#                     optimizer_kwargs={'base_estimator': 'GP'},\n#                     random_state=0,\n#                     verbose=1) \n\n# overdone_control = DeltaYStopper(delta=0.0001)               # stop if the gain of the optimization becomes too small\n# time_limit_control = DeadlineStopper(total_time=60 * 60 * 1) # We impose a time limit\n\n# optimizer.fit(train_features.to_numpy(), df_target.to_numpy().ravel(), callback=[overdone_control, time_limit_control])\n# d=pd.DataFrame(optimizer.cv_results_)\n# best_score = optimizer.best_score_\n# best_score_std = d.iloc[optimizer.best_index_].std_test_score\n# best_params = optimizer.best_params_\n\n# print((\"Bayesian optimization, candidates checked: %d, best CV score: %.3f \"\n#        + u\"\\u00B1\"+\" %.3f\") % (len(optimizer.cv_results_['params']),\n#                                best_score,\n#                                best_score_std))    \n# print('Best parameters:')\n# pprint.pprint(best_params)\n# print()","metadata":{"papermill":{"duration":0.019807,"end_time":"2024-04-28T17:43:11.105152","exception":false,"start_time":"2024-04-28T17:43:11.085345","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.840158Z","iopub.execute_input":"2024-05-12T14:49:24.840421Z","iopub.status.idle":"2024-05-12T14:49:24.847329Z","shell.execute_reply.started":"2024-05-12T14:49:24.840398Z","shell.execute_reply":"2024-05-12T14:49:24.846529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_params = {'application':'binary','verbose':-1,'feature_pre_filter':False,'boosting':'gbdt',\n#     'bagging_fraction': 0.9976747676118453,\n#  'colsample_bytree': 0.11102436264054778,\n#  'feature_fraction': 0.26710140487586775,\n#  'lambda_l1': 16.130951788499626,\n#  'lambda_l2': 65.31083254653984,\n#  'learning_rate': 0.2607586865143843,\n#  'max_depth': int(118.84186862407071),\n#  'min_child_weight': 15.999151640072123,\n#  'min_data_in_leaf': int(1757.9019197260932),\n#  'min_gain_to_split': 1.6565167423234126,\n#  'num_leaves': int(1975.8621766065148),\n#  'subsample': 0.14680112183512767,\n#  'subsample_freq': int(2.7692412551204812)}\n\n\nbest_params_v2 = {'application':'binary','verbose':-1,'feature_pre_filter':False,'boosting':'gbdt','scale_pos_weight':31,\n    'bagging_fraction': 0.8514974164261262,\n 'colsample_bytree': 0.05841975027424331,\n 'feature_fraction': 0.764648199528274,\n 'lambda_l1': 70.5988515667838,\n 'lambda_l2': 81.98932878093925,\n 'learning_rate': 0.06498431363907313,\n 'max_depth': int(60.569593115657376),\n 'min_child_weight': 34.856789045830624,\n 'min_data_in_leaf': int(3268.7703342351533),\n 'min_gain_to_split': 5.866813440708273,\n 'num_leaves': int(2121.97660233422),\n 'subsample': 0.6126362944942335,\n 'subsample_freq': int(4.237641858165943)}\n\n","metadata":{"papermill":{"duration":0.019313,"end_time":"2024-04-28T17:43:11.134209","exception":false,"start_time":"2024-04-28T17:43:11.114896","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.848377Z","iopub.execute_input":"2024-05-12T14:49:24.848634Z","iopub.status.idle":"2024-05-12T14:49:24.861425Z","shell.execute_reply.started":"2024-05-12T14:49:24.848612Z","shell.execute_reply":"2024-05-12T14:49:24.860624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop columns to run model\n# train_features.drop([\n#                     \"education_927M\",\n#                     \"datelastinstal40dpd_247D\",\n#                     \"lastdelinqdate_224D\",\n#                     \"lastrejectdate_50D\",\n#                     \"datelastunpaid_3546854D\",]\n#                     ,axis=1,inplace=True)","metadata":{"papermill":{"duration":0.017982,"end_time":"2024-04-28T17:43:11.161969","exception":false,"start_time":"2024-04-28T17:43:11.143987","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.862418Z","iopub.execute_input":"2024-05-12T14:49:24.862677Z","iopub.status.idle":"2024-05-12T14:49:24.875110Z","shell.execute_reply.started":"2024-05-12T14:49:24.862654Z","shell.execute_reply":"2024-05-12T14:49:24.874248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_features.info()","metadata":{"papermill":{"duration":0.017931,"end_time":"2024-04-28T17:43:11.18968","exception":false,"start_time":"2024-04-28T17:43:11.171749","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.876127Z","iopub.execute_input":"2024-05-12T14:49:24.876432Z","iopub.status.idle":"2024-05-12T14:49:24.885404Z","shell.execute_reply.started":"2024-05-12T14:49:24.876390Z","shell.execute_reply":"2024-05-12T14:49:24.884432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clf = lgb.LGBMClassifier(boosting_type='gbdt',\n#                         objective='binary',\n#                         metric='roc_auc',\n#                          missing=-1,\n#                         n_jobs=1, \n#                         verbose=-1,\n#                         random_state=0)\n\n# model = clf.set_params(**best_params).fit(train_features.to_numpy(), df_target.to_numpy().ravel())\ntrain_data = lgb.Dataset(train_features, df_target)\nmodel = lgb.train(best_params_v2, train_data,num_boost_round=100,)","metadata":{"papermill":{"duration":16.175248,"end_time":"2024-04-28T17:43:27.374688","exception":false,"start_time":"2024-04-28T17:43:11.19944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:49:24.886462Z","iopub.execute_input":"2024-05-12T14:49:24.886690Z","iopub.status.idle":"2024-05-12T14:50:03.140900Z","shell.execute_reply.started":"2024-05-12T14:49:24.886670Z","shell.execute_reply":"2024-05-12T14:50:03.139945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.009416,"end_time":"2024-04-28T17:43:27.3946","exception":false,"start_time":"2024-04-28T17:43:27.385184","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# model_filename = '/kaggle/input/trained-models/lightgbm-v1.pickle'\n# with open(model_filename, 'rb') as f: model = pickle.load(f)","metadata":{"papermill":{"duration":0.018295,"end_time":"2024-04-28T17:43:27.423082","exception":false,"start_time":"2024-04-28T17:43:27.404787","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:03.142109Z","iopub.execute_input":"2024-05-12T14:50:03.143009Z","iopub.status.idle":"2024-05-12T14:50:03.146448Z","shell.execute_reply.started":"2024-05-12T14:50:03.142980Z","shell.execute_reply":"2024-05-12T14:50:03.145750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_features = _general_features(data.test_base).merge(\n        _cb_a_2_features(data.test_cb_a_2),\n        on=[\"case_id\"],\n        how=\"left\",\n    ).merge(\n    pre_test_person_1, on='case_id', how='left'\n).merge(\n    test_static_0_features,on='case_id',how='left'\n).merge(\n    test_static_cb_0_features,on='case_id',how='left'\n).merge(\n    test_credit_bureau_b_1,on='case_id',how='left'\n).merge(\n    test_applprev,on='case_id',how='left'\n).merge(\n    test_deposit,on='case_id',how='left'\n).merge(\n    test_debitcard,on='case_id',how='left'\n).merge(\n    test_other,on='case_id',how='left'\n).merge(\n    test_tax_registry,on='case_id',how='left'\n).pipe(reduce_memory_usage)\n\ntest_features = test_features.drop(columns=missing_cols_dropped)\ntest_features = test_features.drop(columns=unique_cols_dropped)","metadata":{"papermill":{"duration":0.04539,"end_time":"2024-04-28T17:43:27.478909","exception":false,"start_time":"2024-04-28T17:43:27.433519","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:30.380097Z","iopub.execute_input":"2024-05-12T14:50:30.381009Z","iopub.status.idle":"2024-05-12T14:50:30.423198Z","shell.execute_reply.started":"2024-05-12T14:50:30.380974Z","shell.execute_reply":"2024-05-12T14:50:30.422394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_features.shape","metadata":{"papermill":{"duration":0.019492,"end_time":"2024-04-28T17:43:27.508508","exception":false,"start_time":"2024-04-28T17:43:27.489016","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:41.488971Z","iopub.execute_input":"2024-05-12T14:50:41.489315Z","iopub.status.idle":"2024-05-12T14:50:41.495505Z","shell.execute_reply.started":"2024-05-12T14:50:41.489291Z","shell.execute_reply":"2024-05-12T14:50:41.494665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-12T14:50:43.801700Z","iopub.execute_input":"2024-05-12T14:50:43.802462Z","iopub.status.idle":"2024-05-12T14:50:43.808339Z","shell.execute_reply.started":"2024-05-12T14:50:43.802426Z","shell.execute_reply":"2024-05-12T14:50:43.807363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ensure column names are consistent\nassert list(train_features.columns) == list(test_features.columns), \"Training and test data columns do not match.\"","metadata":{"papermill":{"duration":0.018679,"end_time":"2024-04-28T17:43:27.567442","exception":false,"start_time":"2024-04-28T17:43:27.548763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:47.456959Z","iopub.execute_input":"2024-05-12T14:50:47.457583Z","iopub.status.idle":"2024-05-12T14:50:47.461982Z","shell.execute_reply.started":"2024-05-12T14:50:47.457549Z","shell.execute_reply":"2024-05-12T14:50:47.461053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submission_pred = model.predict(test_features.to_numpy())","metadata":{"papermill":{"duration":0.018766,"end_time":"2024-04-28T17:43:27.596821","exception":false,"start_time":"2024-04-28T17:43:27.578055","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:49.534942Z","iopub.execute_input":"2024-05-12T14:50:49.535310Z","iopub.status.idle":"2024-05-12T14:50:49.545199Z","shell.execute_reply.started":"2024-05-12T14:50:49.535284Z","shell.execute_reply":"2024-05-12T14:50:49.544410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submission_pred","metadata":{"papermill":{"duration":0.020139,"end_time":"2024-04-28T17:43:27.688867","exception":false,"start_time":"2024-04-28T17:43:27.668728","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:52.418137Z","iopub.execute_input":"2024-05-12T14:50:52.418955Z","iopub.status.idle":"2024-05-12T14:50:52.426069Z","shell.execute_reply.started":"2024-05-12T14:50:52.418911Z","shell.execute_reply":"2024-05-12T14:50:52.425038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"case_id\": test_features[\"case_id\"].to_numpy(),\n    \"score\": y_submission_pred\n}).set_index('case_id')\nprint(submission)\nsubmission.to_csv(\"./submission.csv\")","metadata":{"papermill":{"duration":0.023143,"end_time":"2024-04-28T17:43:27.722216","exception":false,"start_time":"2024-04-28T17:43:27.699073","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-12T14:50:54.797271Z","iopub.execute_input":"2024-05-12T14:50:54.797648Z","iopub.status.idle":"2024-05-12T14:50:54.806789Z","shell.execute_reply.started":"2024-05-12T14:50:54.797619Z","shell.execute_reply":"2024-05-12T14:50:54.805798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.009875,"end_time":"2024-04-28T17:43:27.742389","exception":false,"start_time":"2024-04-28T17:43:27.732514","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}