{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8248217,"sourceType":"datasetVersion","datasetId":4506020}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Home Credit Model and Submission","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nimport lightgbm as lgb\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, ConfusionMatrixDisplay, classification_report, RocCurveDisplay\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold\nfrom contextlib import suppress\nimport catboost\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb\nfrom statsmodels.stats.weightstats import ztest as ztest\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:03.306686Z","iopub.execute_input":"2024-04-30T01:18:03.307308Z","iopub.status.idle":"2024-04-30T01:18:09.973346Z","shell.execute_reply.started":"2024-04-30T01:18:03.307283Z","shell.execute_reply":"2024-04-30T01:18:09.972440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\ndef set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date, causes issues with other columns ending in D\n        #if col[-1] in (\"D\"):\n            #df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated numeric columns as Float64\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n# Prints every col with missing values > threshold\ndef missing_values(df, threshold = 0.9):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype in ['float64']:\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df\n\n# Returns the range\ndef custom_range_agg(series: pl.series):\n    custom_range = series.max() - series.min()\n    return custom_range","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:16.391407Z","iopub.execute_input":"2024-04-30T01:18:16.392248Z","iopub.status.idle":"2024-04-30T01:18:16.404487Z","shell.execute_reply.started":"2024-04-30T01:18:16.392219Z","shell.execute_reply":"2024-04-30T01:18:16.403573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Used in training preprocess to compare sum and range aggregated features against each other, printing the percent of the range column that either equals 0 or the value of the sum column\n# Drop range feature if percent > 0.9. Function will also replace nulls with 0s only for range columns \ndef compare_cols(df):\n    range_list = [element for e in df.columns for element in e.split() if element.endswith(\"_range\")]\n    sum_list = [name.replace('range', 'sum') for name in range_list]\n    length = len(df)\n\n    df=df.with_columns(*[pl.col(col).fill_null(strategy='zero') for col in range_list])\n    \n    for i in range(len(range_list)):\n        df = df.with_columns(check=\n        pl.when((df[range_list[i]] == df[sum_list[i]]) | (df[range_list[i]] == 0)).then(1).otherwise(0))\n\n        percent = df.select(pl.sum(\"check\")).item() / length\n        print(f\"Percent equal or 0 for {range_list[i]} = {percent:.3f}\")\n        \n    df = df.drop(\"check\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:29.850202Z","iopub.execute_input":"2024-04-30T01:18:29.851015Z","iopub.status.idle":"2024-04-30T01:18:29.860729Z","shell.execute_reply.started":"2024-04-30T01:18:29.850977Z","shell.execute_reply":"2024-04-30T01:18:29.859667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken directly from other competition notebooks\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:32.601745Z","iopub.execute_input":"2024-04-30T01:18:32.602151Z","iopub.status.idle":"2024-04-30T01:18:32.619181Z","shell.execute_reply.started":"2024-04-30T01:18:32.602120Z","shell.execute_reply":"2024-04-30T01:18:32.617737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate joined test data \n#### Part 1\nPreprocessed the training csvs in the same way, lists of columns that are dropped were manually taken from what was dropped in training based on excessive missing values. Used .drop(errors='ignore') to handle situations where hidden test set has different columns. ","metadata":{}},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway +  \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:36.265131Z","iopub.execute_input":"2024-04-30T01:18:36.265484Z","iopub.status.idle":"2024-04-30T01:18:36.406670Z","shell.execute_reply.started":"2024-04-30T01:18:36.265456Z","shell.execute_reply":"2024-04-30T01:18:36.405738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional depth=1 files\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\n# test_debitcard_1 = pl.read_csv(pathway + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:39.271984Z","iopub.execute_input":"2024-04-30T01:18:39.272374Z","iopub.status.idle":"2024-04-30T01:18:39.300816Z","shell.execute_reply.started":"2024-04-30T01:18:39.272344Z","shell.execute_reply":"2024-04-30T01:18:39.299940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use aggregation functions in tables with depth >=1\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:41.882538Z","iopub.execute_input":"2024-04-30T01:18:41.883278Z","iopub.status.idle":"2024-04-30T01:18:41.905865Z","shell.execute_reply.started":"2024-04-30T01:18:41.883245Z","shell.execute_reply":"2024-04-30T01:18:41.905071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"), \n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:46.539791Z","iopub.execute_input":"2024-04-30T01:18:46.540215Z","iopub.status.idle":"2024-04-30T01:18:46.555452Z","shell.execute_reply.started":"2024-04-30T01:18:46.540186Z","shell.execute_reply":"2024-04-30T01:18:46.554345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:49.211159Z","iopub.execute_input":"2024-04-30T01:18:49.211481Z","iopub.status.idle":"2024-04-30T01:18:49.234999Z","shell.execute_reply.started":"2024-04-30T01:18:49.211458Z","shell.execute_reply":"2024-04-30T01:18:49.234069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n# Based on training preprocessing, shoudl be 194 - 1 = 193 cols\ndrop_cols = ['clientscnt_136L', 'datelastinstal40dpd_247D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'interestrategrace_34L', 'isbidproductrequest_292L', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'payvacationpostpone_4187118D', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'riskassesment_302T', 'riskassesment_940T', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:51.442626Z","iopub.execute_input":"2024-04-30T01:18:51.442964Z","iopub.status.idle":"2024-04-30T01:18:51.488853Z","shell.execute_reply.started":"2024-04-30T01:18:51.442938Z","shell.execute_reply":"2024-04-30T01:18:51.487872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1,test_credit_bureau_b_1,test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats   \ngc.collect()  ","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:53.846665Z","iopub.execute_input":"2024-04-30T01:18:53.847244Z","iopub.status.idle":"2024-04-30T01:18:53.959414Z","shell.execute_reply.started":"2024-04-30T01:18:53.847215Z","shell.execute_reply":"2024-04-30T01:18:53.958461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Part 2","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:56.625849Z","iopub.execute_input":"2024-04-30T01:18:56.626637Z","iopub.status.idle":"2024-04-30T01:18:56.696987Z","shell.execute_reply.started":"2024-04-30T01:18:56.626606Z","shell.execute_reply":"2024-04-30T01:18:56.696263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:18:59.685241Z","iopub.execute_input":"2024-04-30T01:18:59.685606Z","iopub.status.idle":"2024-04-30T01:18:59.694686Z","shell.execute_reply.started":"2024-04-30T01:18:59.685579Z","shell.execute_reply":"2024-04-30T01:18:59.693620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    custom_range_agg(pl.col(\"annuity_853A\")).alias('annuity_853A_range'),\n    pl.col(\"byoccupationinc_3656910L\").sum().alias(\"byoccupationinc_3656910L_sum\"),\n    custom_range_agg(pl.col(\"byoccupationinc_3656910L\")).alias('byoccupationinc_3656910L_range'),\n    pl.col(\"childnum_21L\").sum().alias(\"childnum_21L_sum\"),\n    custom_range_agg(pl.col(\"childnum_21L\")).alias('childnum_21L_range'),\n    pl.col(\"credacc_credlmt_575A\").sum().alias(\"credacc_credlmt_575A_sum\"),\n    custom_range_agg(pl.col(\"credacc_credlmt_575A\")).alias('credacc_credlmt_575A_range'),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    custom_range_agg(pl.col(\"downpmt_134A\")).alias('downpmt_134A_range'),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    custom_range_agg(pl.col(\"mainoccupationinc_437A\")).alias('mainoccupationinc_437A_range'),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    custom_range_agg(pl.col(\"pmtnum_8L\")).alias('pmtnum_8L_range'),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"),\n    custom_range_agg(pl.col(\"tenor_203L\")).alias('tenor_203L_range'))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:02.011849Z","iopub.execute_input":"2024-04-30T01:19:02.012467Z","iopub.status.idle":"2024-04-30T01:19:02.027553Z","shell.execute_reply.started":"2024-04-30T01:19:02.012436Z","shell.execute_reply":"2024-04-30T01:19:02.026684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"),\n    custom_range_agg(pl.col(\"amount_4527230A\")).alias('amount_4527230A_range'))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:04.469956Z","iopub.execute_input":"2024-04-30T01:19:04.470752Z","iopub.status.idle":"2024-04-30T01:19:04.481124Z","shell.execute_reply.started":"2024-04-30T01:19:04.470724Z","shell.execute_reply":"2024-04-30T01:19:04.480288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = compare_cols(test_applprev_1_feats_1)\ntest_tax_registry_a_1_feats = compare_cols(test_tax_registry_a_1_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:06.818814Z","iopub.execute_input":"2024-04-30T01:19:06.819673Z","iopub.status.idle":"2024-04-30T01:19:06.837168Z","shell.execute_reply.started":"2024-04-30T01:19:06.819643Z","shell.execute_reply":"2024-04-30T01:19:06.836199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    custom_range_agg(pl.col(\"monthlyinstlamount_332A\")).alias('monthlyinstlamount_332A_range'),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    custom_range_agg(pl.col(\"monthlyinstlamount_674A\")).alias('monthlyinstlamount_674A_range'),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    custom_range_agg(pl.col(\"numberofinstls_229L\")).alias('numberofinstls_229L_range'),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    custom_range_agg(pl.col(\"numberofoverdueinstlmax_1151L\")).alias('numberofoverdueinstlmax_1151L_range'),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"),\n    custom_range_agg(pl.col(\"totalamount_6A\")).alias('totalamount_6A_range'))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))\n\ntest_credit_bureau_a_1_feats = compare_cols(test_credit_bureau_a_1_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:10.097783Z","iopub.execute_input":"2024-04-30T01:19:10.098195Z","iopub.status.idle":"2024-04-30T01:19:10.115090Z","shell.execute_reply.started":"2024-04-30T01:19:10.098165Z","shell.execute_reply":"2024-04-30T01:19:10.114080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','amount_4917619A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:13.301044Z","iopub.execute_input":"2024-04-30T01:19:13.301826Z","iopub.status.idle":"2024-04-30T01:19:13.317873Z","shell.execute_reply.started":"2024-04-30T01:19:13.301798Z","shell.execute_reply":"2024-04-30T01:19:13.317069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1,test_credit_bureau_a_1\ndel test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:16.268674Z","iopub.execute_input":"2024-04-30T01:19:16.269406Z","iopub.status.idle":"2024-04-30T01:19:16.381112Z","shell.execute_reply.started":"2024-04-30T01:19:16.269376Z","shell.execute_reply":"2024-04-30T01:19:16.380052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Part 3","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 2\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:19.776485Z","iopub.execute_input":"2024-04-30T01:19:19.777304Z","iopub.status.idle":"2024-04-30T01:19:19.795765Z","shell.execute_reply.started":"2024-04-30T01:19:19.777274Z","shell.execute_reply":"2024-04-30T01:19:19.794828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:22.785801Z","iopub.execute_input":"2024-04-30T01:19:22.786506Z","iopub.status.idle":"2024-04-30T01:19:22.857674Z","shell.execute_reply.started":"2024-04-30T01:19:22.786475Z","shell.execute_reply":"2024-04-30T01:19:22.856768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_1073P\")).alias('pmts_dpd_1073P_range'),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_303P\")).alias('pmts_dpd_303P_range'),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1140A\")).alias('pmts_overdue_1140A_range'),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1152A\")).alias('pmts_overdue_1152A_range'))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:25.239796Z","iopub.execute_input":"2024-04-30T01:19:25.240662Z","iopub.status.idle":"2024-04-30T01:19:25.252248Z","shell.execute_reply.started":"2024-04-30T01:19:25.240630Z","shell.execute_reply":"2024-04-30T01:19:25.251108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_2_feats = compare_cols(test_credit_bureau_a_2_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:28.257007Z","iopub.execute_input":"2024-04-30T01:19:28.257873Z","iopub.status.idle":"2024-04-30T01:19:28.264890Z","shell.execute_reply.started":"2024-04-30T01:19:28.257845Z","shell.execute_reply":"2024-04-30T01:19:28.263860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:30.921254Z","iopub.execute_input":"2024-04-30T01:19:30.921659Z","iopub.status.idle":"2024-04-30T01:19:30.932462Z","shell.execute_reply.started":"2024-04-30T01:19:30.921632Z","shell.execute_reply":"2024-04-30T01:19:30.931390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2, test_person_2, test_credit_bureau_a_2\ndel test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:33.614594Z","iopub.execute_input":"2024-04-30T01:19:33.614935Z","iopub.status.idle":"2024-04-30T01:19:33.728130Z","shell.execute_reply.started":"2024-04-30T01:19:33.614908Z","shell.execute_reply":"2024-04-30T01:19:33.727070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n# Convert back to polars for datetime \njoin_test = pl.from_pandas(join_test)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:36.401872Z","iopub.execute_input":"2024-04-30T01:19:36.402580Z","iopub.status.idle":"2024-04-30T01:19:36.445370Z","shell.execute_reply.started":"2024-04-30T01:19:36.402549Z","shell.execute_reply":"2024-04-30T01:19:36.444464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_test.head() ","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:38.662808Z","iopub.execute_input":"2024-04-30T01:19:38.663186Z","iopub.status.idle":"2024-04-30T01:19:38.682292Z","shell.execute_reply.started":"2024-04-30T01:19:38.663158Z","shell.execute_reply":"2024-04-30T01:19:38.681211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Join Test Data","metadata":{}},{"cell_type":"code","source":"test = join_test.with_columns(pl.col('date_decision').cast(pl.Date))\n# Feature engineer date columns\ndate_list = ['datefirstoffer_1144D', 'datelastunpaid_3546854D', 'dtlastpmtallstes_4499206D', 'firstclxcampaign_1125D', 'lastdelinqdate_224D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'birthdate_574D', 'responsedate_1012D', 'responsedate_4527233D', 'responsedate_4917613D', 'empl_employedfrom_271D',\n            'firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D','birth_259D']\n\n# Calculates date_diff as date_decision minus each date column from above\nfor col in date_list:\n    test = test.with_columns(pl.col(col).cast(pl.Date))\n    test = test.with_columns(\n        ((pl.col(\"date_decision\") - pl.col(col)) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(f\"{col}_diff\"))\n\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\n# Convert _range features to Float64\nfor col in test.columns:\n        if col[-5:-1] in (\"range\"):\n            test = test.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            \ndrop_list = ['date_decision','MONTH'] + date_list\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(drop_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test, dfs\ngc.collect()\n\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:41.846867Z","iopub.execute_input":"2024-04-30T01:19:41.847265Z","iopub.status.idle":"2024-04-30T01:19:42.017925Z","shell.execute_reply.started":"2024-04-30T01:19:41.847235Z","shell.execute_reply":"2024-04-30T01:19:42.016901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noticed these numeric variables were parsed as strings, changed them all to float (same as training preprocessing)\nnum_list = ['numinstlswithdpd5_4187116L', 'numinstmatpaidtearly2d_4499204L','numinstpaid_4499208L','numinstpaidearly3dest_4493216L','numinstpaidearly5dest_4493211L',\n           'numinstpaidearly5dobd_4499205L','numinstpaidearlyest_4493214L','numinstpaidlastcontr_4325080L','numinstregularpaidest_4493210L','numinsttopaygrest_4493213L','numinstunpaidmaxest_4493212L',\n           'contractssum_5085716L','days120_123L','days180_256L','days360_512L','firstquarter_103L','fourthquarter_440L','numberofqueries_373L','pmtscount_423L','secondquarter_766L','thirdquarter_1082L',\n           'days30_165L','days90_310L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:45.301041Z","iopub.execute_input":"2024-04-30T01:19:45.301392Z","iopub.status.idle":"2024-04-30T01:19:45.316639Z","shell.execute_reply.started":"2024-04-30T01:19:45.301360Z","shell.execute_reply":"2024-04-30T01:19:45.315733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same as training preprocessing (features either all the same or categorical variables with unique categories > 150)\ndrop_list = ['commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L','addres_zip_823M',\n            'lastapprcommoditytypec_5251766M','lastrejectcommodtypec_5251769M','previouscontdistrict_112M', 'addres_district_368M']\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\n# Drop rows with all na\ntest.dropna(axis=1, how='all', inplace=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:48.189367Z","iopub.execute_input":"2024-04-30T01:19:48.190386Z","iopub.status.idle":"2024-04-30T01:19:48.249153Z","shell.execute_reply.started":"2024-04-30T01:19:48.190337Z","shell.execute_reply":"2024-04-30T01:19:48.248073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Imputation","metadata":{}},{"cell_type":"code","source":"\"\"\"\nprint(np.count_nonzero(test.isnull()))\n# Impute missing values, 0 missing values after imputation\ntest = imputer(test)\nprint(np.count_nonzero(test.isnull()))\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See boolean columns\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:52.466441Z","iopub.execute_input":"2024-04-30T01:19:52.466998Z","iopub.status.idle":"2024-04-30T01:19:52.476211Z","shell.execute_reply.started":"2024-04-30T01:19:52.466969Z","shell.execute_reply":"2024-04-30T01:19:52.475171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify dtypes are correct\n#pd.set_option('display.max_rows', None)\n#test.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ncat_cols = test.select_dtypes(include=['category', 'object']).columns.tolist()\n# Create dummies for all cat columns, not dropping first to keep column names same as training\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = reduce_mem_usage(test)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:26.871745Z","iopub.execute_input":"2024-04-30T01:21:26.872145Z","iopub.status.idle":"2024-04-30T01:21:26.995541Z","shell.execute_reply.started":"2024-04-30T01:21:26.872116Z","shell.execute_reply":"2024-04-30T01:21:26.994358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Train data","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv('/kaggle/input/training/train251_noimpute.csv').pipe(set_table_dtypes).pipe(convert_strings)\nfor col in train.columns:\n        if col[-5:-1] in (\"range\"):\n            train = train.with_columns(pl.col(col).cast(pl.Float64).alias(col))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:19:56.832140Z","iopub.execute_input":"2024-04-30T01:19:56.832782Z","iopub.status.idle":"2024-04-30T01:20:13.336854Z","shell.execute_reply.started":"2024-04-30T01:19:56.832753Z","shell.execute_reply":"2024-04-30T01:20:13.336016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert polars df to pandas so pandas specific methods/attributes work later, seems more memory efficient to load as pl and convert to pd than load as pd\ntrain = train.to_pandas()\n# Get list of ids for submission file\nids = test['case_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:31.012372Z","iopub.execute_input":"2024-04-30T01:21:31.012719Z","iopub.status.idle":"2024-04-30T01:21:34.770100Z","shell.execute_reply.started":"2024-04-30T01:21:31.012693Z","shell.execute_reply":"2024-04-30T01:21:34.769095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Make sure all cols are numeric if data was converted to dummies\nconvert_cols = train.select_dtypes(include=['category', 'object']).columns.tolist()\nfor col in convert_cols:\n    train[col]= pd.to_numeric(train[col])\ntrain.shape\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary Stats, Corr Heatmap, Additional feature analysis","metadata":{}},{"cell_type":"code","source":"# Summary Stats\npd.options.display.float_format = \"{:.2f}\".format\nimpt_list = ['incometype_1044T', 'lastapprcommoditycat_1041M', 'price_1097A', 'maritalst_385M', 'lastcancelreason_561M',\n            'pmts_overdue_1140A_sum', 'lastrejectdate_50D_diff', 'pctinstlsallpaidlate4d_3546849L', 'totalamount_6A_sum',\n            'pmtnum_254L', 'birth_259D_diff', 'avgdpdtolclosure24_3658938P', 'lastrejectcommoditycat_161M', 'education_1103M', 'sex_738L']\nimpt_num_list = ['price_1097A', 'pmts_overdue_1140A_sum', 'lastrejectdate_50D_diff', 'pctinstlsallpaidlate4d_3546849L', 'totalamount_6A_sum',\n                'pmtnum_254L', 'birth_259D_diff', 'avgdpdtolclosure24_3658938P']\nimpt_cat_list = [col for col in impt_list if col not in impt_num_list]\nimpt_list.sort()\ntrain[impt_list].describe()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:34.950457Z","iopub.execute_input":"2024-04-30T01:21:34.950780Z","iopub.status.idle":"2024-04-30T01:21:35.661100Z","shell.execute_reply.started":"2024-04-30T01:21:34.950754Z","shell.execute_reply":"2024-04-30T01:21:35.660082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation heatmap of impt numeric features\nimpt_num_list.sort()\nmatrix = train[impt_num_list].corr().round(2)\nsns.heatmap(matrix, annot=True, vmax=1, vmin=-1, center=0, cmap='vlag')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:38.783884Z","iopub.execute_input":"2024-04-30T01:21:38.784241Z","iopub.status.idle":"2024-04-30T01:21:39.617918Z","shell.execute_reply.started":"2024-04-30T01:21:38.784215Z","shell.execute_reply":"2024-04-30T01:21:39.616981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impt numeric cols, mean based on target label\nfor col in impt_num_list:\n    print(train.groupby('target')[col].mean())","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:46.336379Z","iopub.execute_input":"2024-04-30T01:21:46.336727Z","iopub.status.idle":"2024-04-30T01:21:46.568319Z","shell.execute_reply.started":"2024-04-30T01:21:46.336699Z","shell.execute_reply":"2024-04-30T01:21:46.567385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_df = pd.DataFrame({'price_1097A':[34297.68, 39603.99],'pmts_overdue_1140A_sum':[49231.74, 83557.00],'lastrejectdate_50D_diff':[846.42, 573.49],\n                       'pctinstlsallpaidlate4d_3546849L':[0.11, 0.24] , 'totalamount_6A_sum':[340655.96, 203042.52],'pmtnum_254L':[17.01, 18.95], \n                       'birth_259D_diff':[16326.43, 14734.13], 'avgdpdtolclosure24_3658938P':[42.76, 137.11]})\nmean_df = mean_df.reindex(sorted(mean_df.columns), axis=1)\nmean_df","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:49.499282Z","iopub.execute_input":"2024-04-30T01:21:49.499628Z","iopub.status.idle":"2024-04-30T01:21:49.514474Z","shell.execute_reply.started":"2024-04-30T01:21:49.499602Z","shell.execute_reply":"2024-04-30T01:21:49.513515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate P-values for one-sided Z tests of means \nimpt_num_list.sort()\nfor col in impt_num_list:\n    temp = train[train[[col]].notna().any(axis=1)]\n    temp1 = temp.loc[(temp['target'] ==0)][col].values\n    temp2 = temp.loc[(temp['target'] ==1)][col].values\n    z, p = ztest(temp1, temp2, value=0, alternative='larger', usevar='unequal')\n    z1, p1 = ztest(temp1, temp2, value=0, alternative='smaller', usevar='unequal')\n    print(f'P-Value for {col}: {min(p,p1)}')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:21:52.605395Z","iopub.execute_input":"2024-04-30T01:21:52.606069Z","iopub.status.idle":"2024-04-30T01:22:07.439162Z","shell.execute_reply.started":"2024-04-30T01:21:52.606019Z","shell.execute_reply":"2024-04-30T01:22:07.438231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.float_format = \"{:.4f}\".format\npd.set_option('display.max_rows', 20)\nimpt_cat_list.sort()\n# display df showing target percentage and count for each impt cat feature\nfor col in impt_cat_list:\n    a = train.groupby(col, observed=True)['target'].sum()\n    b = train.groupby(col, observed=True)['target'].count()\n    temp = pd.DataFrame(a/b)\n    temp.insert(1, \"count\", b.values, True)\n    display(temp.sort_values('target',ascending=False).head(20))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:23:05.958381Z","iopub.execute_input":"2024-04-30T01:23:05.959210Z","iopub.status.idle":"2024-04-30T01:23:06.406338Z","shell.execute_reply.started":"2024-04-30T01:23:05.959174Z","shell.execute_reply":"2024-04-30T01:23:06.405334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='avgdpdtolclosure24_3658938P', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='birth_259D_diff', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='lastrejectdate_50D_diff', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='pctinstlsallpaidlate4d_3546849L', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='pmtnum_254L', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='pmts_overdue_1140A_sum', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='target', y='price_1097A', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsns.boxplot(x='target', y='totalamount_6A_sum', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.violinplot(x='target', y='totalamount_6A_sum', data=train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(impt_num_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=8, figsize=(30, 5))\n\nfor i, feature in enumerate(impt_num_list):\n    sns.boxplot(x='target', y=feature, data=train, ax=axes[i])\n    axes[i].set_title(feature)\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T01:24:28.514231Z","iopub.execute_input":"2024-04-30T01:24:28.515273Z","iopub.status.idle":"2024-04-30T01:24:32.500955Z","shell.execute_reply.started":"2024-04-30T01:24:28.515225Z","shell.execute_reply":"2024-04-30T01:24:32.500054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=4, ncols=2, figsize=(35, 30))\n\nsns.countplot(x='education_1103M', data=train, ax=axes[0,0])\naxes[0,0].set_title('education_1103M')\n\nsns.countplot(x='incometype_1044T', data=train, ax=axes[0,1])\naxes[0,1].set_title('incometype_1044T')\naxes[0,1].set_xticklabels(axes[0,1].get_xticklabels(), rotation=45, ha='right')\n\nsns.countplot(x='lastapprcommoditycat_1041M', data=train, ax=axes[1,0])\naxes[1,0].set_title('lastapprcommoditycat_1041M')\nunique_categories = train['lastapprcommoditycat_1041M'].unique()\noffset=0.5\naxes[1,0].set_xticks([i + offset for i in range(len(unique_categories))])\naxes[1,0].set_xticklabels(unique_categories, rotation=45, ha='right')\n\nsns.countplot(x='lastcancelreason_561M', data=train, ax=axes[1,1])\naxes[1,1].set_title('lastcancelreason_561M')\nunique_categories2 = train['lastcancelreason_561M'].unique()\noffset_cancel = 2\naxes[1,1].set_xticks([i + offset_cancel for i in range(len(unique_categories2))])\naxes[1,1].set_xticklabels(unique_categories2, rotation=45, ha='right')\n\nsns.countplot(x='lastrejectcommoditycat_161M', data=train, ax=axes[2,0])\naxes[2,0].set_title('lastrejectcommoditycat_161M')\nunique_categories3 = train['lastrejectcommoditycat_161M'].unique()\noffset_reject = 0.5\naxes[2,0].set_xticks([i + offset_reject for i in range(len(unique_categories3))])\naxes[2,0].set_xticklabels(unique_categories3, rotation=45, ha='right')\n\nsns.countplot(x='maritalst_385M', data=train, ax=axes[2,1])\naxes[2,1].set_title('maritalst_385M')\n\nsns.countplot(x='sex_738L', data=train, ax=axes[3,0])\naxes[3,0].set_title('sex_738L')\n\nfig.delaxes(axes[3,1])\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T02:21:44.401669Z","iopub.execute_input":"2024-04-30T02:21:44.402050Z","iopub.status.idle":"2024-04-30T02:21:47.921848Z","shell.execute_reply.started":"2024-04-30T02:21:44.402004Z","shell.execute_reply":"2024-04-30T02:21:47.921008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Additional Preprocessing","metadata":{}},{"cell_type":"code","source":"train = reduce_mem_usage(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.set_option('display.max_rows', None)\n#train.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only select common columns to use\ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest=test[common_columns]\n\n# Subset train with only columns seen in test + target\ntrain = train[common_columns+['target']]\ntrain.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Fit on stratified sample\n# Used for testing code\ntrain_sample = train.groupby('target', group_keys=False).apply(lambda x: x.sample(frac=0.01)).reset_index(drop=True)\ntrain_sample.head()\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.loc[:,'target'].to_frame('target')\nX = train.drop(['target'], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check target distribution is same after all the preprocessing/sampling\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(),4))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do not include case_id, or week_num as numeric \nnumeric_cols = test.select_dtypes(include=['number']).columns.tolist()\nnumeric_cols.remove('case_id')\nnumeric_cols.remove('WEEK_NUM')\n#print(numeric_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nwarnings.filterwarnings(\"default\")\n\nX.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"code","source":"# Drop case_id and week_num from features\nweeks = X[\"WEEK_NUM\"]\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n# Sort columns in alphabetical order for training so columns match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\n\nprint(X_feats.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include=['category', 'object']).columns.tolist()\n#print(cat_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_feats[cat_cols] = X_feats[cat_cols].astype(str)\ntest[cat_cols] = test[cat_cols].astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nfitted_models_cat = []\nfitted_models_lgb = []\n#fitted_models_xgb = []\n\ncv_scores_cat = []\ncv_scores_lgb = []\n#cv_scores_xgb = []\n\n# Note: Currently running with GPU, change params if no accelerator\ngrid_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2500,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"verbose\": -1,\n    'device':'gpu',\n    #'device': 'cpu'\n}\n\n\"\"\"\ngrid_params2 = {\n    \"booster\": \"gbtree\",\n    \"objective\": \"binary:logistic\",\n    \"eval_metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"alpha\": 0.1,  \n    \"lambda\": 10,  \n    \"tree_method\": 'gpu_hist',\n    #\"tree_method\": 'auto',\n    \"random_state\": 123,\n    \"verbosity\": 0,\n    \"enable_categorical\":True,\n}\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nwarnings.filterwarnings(\"ignore\")\n\n# Ensemble model inspired by other public competition notebooks\n# Group by weeks to account for gini instability weekly metric\nfor idx_train, idx_valid in cv.split(X_feats, y, groups=weeks): \n    # Split training in every fold by setting the split indexes\n    X_train, y_train = X_feats.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X_feats.iloc[idx_valid], y.iloc[idx_valid]\n    # Pool used for CatBoost\n    train_pool = Pool(X_train, y_train,cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid,cat_features=cat_cols)\n    \n    clf = CatBoostClassifier(\n    eval_metric='AUC', task_type='GPU', # Change if not using GPU\n    learning_rate=0.05, iterations = 5000, # default 1k\n    random_seed=3107)  \n    clf.fit(train_pool, eval_set=val_pool,verbose=300)\n    # Append model and auc_score to their appropriate list\n    fitted_models_cat.append(clf)\n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n    \n    X_train[cat_cols] = X_train[cat_cols].astype(\"category\")\n    X_valid[cat_cols] = X_valid[cat_cols].astype(\"category\")\n    \n    # Train lgbm model, stopping if no improvement in auc_score in 200 rounds\n    model = lgb.LGBMClassifier(**grid_params)\n    model.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(200)] )\n    # Again, append the model and auc_score\n    fitted_models_lgb.append(model)\n    y_pred_valid = model.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_lgb.append(auc_score)\n    \n    \"\"\"\n    # Final XGB model with same early stopping \n    model2 = xgb.XGBClassifier(**grid_params2)\n    model2.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        early_stopping_rounds=100, verbose=False)\n    # Again, append model and auc_score\n    fitted_models_xgb.append(model2)\n    y_pred_valid = model2.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_xgb.append(auc_score)\n    \"\"\"\n    \n    del clf, model\n    gc.collect()\n    \nprint(\"CV CAT AUC scores: \", cv_scores_cat)\nprint(\"Maximum CV CAT AUC score: \", max(cv_scores_cat))\n\nprint(\"CV LGB AUC scores: \", cv_scores_lgb)\nprint(\"Maximum CV LGB AUC score: \", max(cv_scores_lgb))\n\n\"\"\"\nprint(\"CV XGB AUC scores: \", cv_scores_xgb)\nprint(\"Maximum CV XGB AUC score: \", max(cv_scores_xgb))\n\"\"\"\n\nwarnings.filterwarnings(\"default\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the best cv model\nclass VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators[:5]]    \n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        y_preds += [estimator.predict(X) for estimator in self.estimators[5:10]]\n        # Can test including or not, would give more weight to CatBoost and LGBM predicts\n        #y_preds+=y_preds \n        \n        # y_preds += [estimator.predict(X) for estimator in self.estimators[10:]]\n        return np.mean(y_preds, axis=0)\n    \n    # main use case for this class \n    def predict_proba(self, X):\n        # predict_proba for CatBoost\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]    \n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        # predict_proba for LGBM\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[5:10]]\n        # Can test including or not, would give more weight to CatBoost and LGBM predicts\n        #y_preds+=y_preds \n        \n        # predict_proba for XGB\n        # y_preds += [estimator.predict_proba(X) for estimator in self.estimators[10:]]\n        return np.mean(y_preds, axis=0)\n    \n\nmodel = VotingModel(fitted_models_cat + fitted_models_lgb)\nmodel_lgb = fitted_models_lgb[np.argmax(cv_scores_lgb)]\nmodel_cat = fitted_models_cat[np.argmax(cv_scores_cat)]\n# model_xgb = fitted_models_xgb[np.argmax(cv_scores_xgb)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del fitted_models_cat, fitted_models_lgb\ngc.collect() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion Matrix and Classification Report","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\ny_pred = model.predict(X_feats)\ny_pred1 = [1 if num > 0.50 else 0 for num in y_pred]\n\ntarget_names= ['No Default', 'Default']\nmatrix = confusion_matrix(y, y_pred1, normalize='true')\ncm_display = ConfusionMatrixDisplay(confusion_matrix= matrix, display_labels=target_names).plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print out the report\nprint(classification_report(y, y_pred1, target_names = target_names))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot ROC AUC curve for all training data\n# I prefer skplt.metrics.plot_roc_curve but there is an error with the scipy version and would necessitate creating a custom environment.\n\nfpr, tpr, threshold = metrics.roc_curve(y_valid, y_pred_valid)\ndisplay = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=auc_score)\ndisplay.plot()\nplt.plot([0, 1], [0, 1],'r--')\nplt.title('ROC-AUC Curve for Validation Data')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_pred, y_pred1\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"code","source":"# For lgb model, ranks features based on number of splits\nlgb.plot_importance(model_lgb, figsize=(20,20), max_num_features = 20)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb.plot_importance(model_lgb, figsize=(20,20), importance_type='gain', max_num_features =20)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_feature_importance(importance,columns,model_name):\n    # Create arrays for feature importance and feature names\n    feature_importance = np.array(importance)\n    feature_names = np.array(columns)\n    \n    # Create a df using a dictionary\n    data={'feature_names':feature_names,'feature_importance':feature_importance}\n    temp_df = pd.DataFrame(data)\n    \n    #Sort the df in order of decreasing feature importance\n    temp_df.sort_values(by=['feature_importance'], ascending=False,inplace=True)\n    \n    plt.figure(figsize=(20,20))\n    # Plot only top 20 features \n    sns.barplot(x=temp_df['feature_importance'][:20], y=temp_df['feature_names'][:20])\n\n    plt.title(model_name)\n    plt.xlabel('Feature Importance')\n    plt.ylabel('Feature Names')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculates the importance of each feature by comparing the model’s performance with and without that feature. \n# Difference in the loss function obtained when including the feature versus excluding it from the model. The greater the difference, the more important the feature.\nplot_feature_importance(model_cat.get_feature_importance(type='PredictionValuesChange'), X_train.columns, \"CATBOOST\")\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Gain is derived from a complicated formula but can be summarized as the relative contribution of each feature to the model.\n# Total gain is just the sum of gains across all splits.\n# https://xgboost.readthedocs.io/en/latest/tutorials/model.html\nxgb_impt = model_xgb.get_booster().get_score(importance_type='total_gain')\nkeys = list(xgb_impt.keys())\nvalues = list(xgb_impt.values())\n\nplot_feature_importance(values, keys, \"XGB Total Gain\")\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nxgb_impt = model_xgb.get_booster().get_score(importance_type='gain')\nkeys = list(xgb_impt.keys())\nvalues = list(xgb_impt.values())\n\nplot_feature_importance(values, keys, \"XGB Gain\")\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_feats, X_train, X_valid, y_train, y_valid, y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# Sort columns alphabetically to match loaded model\ntest = test.reindex(sorted(test.columns), axis=1)\n\npredictions = model.predict_proba(test.drop(['case_id', 'WEEK_NUM'], axis=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id': ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}