{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":50160,"databundleVersionId":7921029},{"sourceType":"datasetVersion","sourceId":8274557,"datasetId":4506020,"databundleVersionId":8403629}],"dockerImageVersionId":30665,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Home Credit Model and Submission\nNote: May need to disable some of the graphs if memory issues","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nimport lightgbm as lgb\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, ConfusionMatrixDisplay, classification_report, RocCurveDisplay\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold\nfrom contextlib import suppress\nimport catboost\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb\nfrom statsmodels.stats.weightstats import ztest as ztest\nfrom IPython.display import display","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.253194Z","iopub.execute_input":"2024-04-30T19:14:47.254282Z","iopub.status.idle":"2024-04-30T19:14:47.262720Z","shell.execute_reply.started":"2024-04-30T19:14:47.254235Z","shell.execute_reply":"2024-04-30T19:14:47.261540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\ndef set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date, causes issues with other columns ending in D\n        #if col[-1] in (\"D\"):\n            #df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated numeric columns as Float64\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n# Prints every col with missing values > threshold\ndef missing_values(df, threshold = 0.9):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype in ['float64']:\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df\n\n# Returns the range\ndef custom_range_agg(series: pl.series):\n    custom_range = series.max() - series.min()\n    return custom_range","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.266276Z","iopub.execute_input":"2024-04-30T19:14:47.266569Z","iopub.status.idle":"2024-04-30T19:14:47.280995Z","shell.execute_reply.started":"2024-04-30T19:14:47.266544Z","shell.execute_reply":"2024-04-30T19:14:47.279836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Used in training preprocess to compare sum and range aggregated features against each other, printing the percent of the range column that either equals 0 or the value of the sum column\n# Drop range feature if percent > 0.9. Function will also replace nulls with 0s only for range columns \ndef compare_cols(df):\n    range_list = [element for e in df.columns for element in e.split() if element.endswith(\"_range\")]\n    sum_list = [name.replace('range', 'sum') for name in range_list]\n    length = len(df)\n\n    df=df.with_columns(*[pl.col(col).fill_null(strategy='zero') for col in range_list])\n    \n    for i in range(len(range_list)):\n        df = df.with_columns(check=\n        pl.when((df[range_list[i]] == df[sum_list[i]]) | (df[range_list[i]] == 0)).then(1).otherwise(0))\n\n        percent = df.select(pl.sum(\"check\")).item() / length\n        print(f\"Percent equal or 0 for {range_list[i]} = {percent:.3f}\")\n        \n    df = df.drop(\"check\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.283477Z","iopub.execute_input":"2024-04-30T19:14:47.283850Z","iopub.status.idle":"2024-04-30T19:14:47.295222Z","shell.execute_reply.started":"2024-04-30T19:14:47.283815Z","shell.execute_reply":"2024-04-30T19:14:47.294244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken directly from other competition notebooks\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.296661Z","iopub.execute_input":"2024-04-30T19:14:47.297399Z","iopub.status.idle":"2024-04-30T19:14:47.312650Z","shell.execute_reply.started":"2024-04-30T19:14:47.297364Z","shell.execute_reply":"2024-04-30T19:14:47.311721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate joined test data \n#### Part 1\nPreprocessed the training csvs in the same way, lists of columns that are dropped were manually taken from what was dropped in training based on excessive missing values. Used .drop(errors='ignore') to handle situations where hidden test set has different columns. ","metadata":{}},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway +  \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.313661Z","iopub.execute_input":"2024-04-30T19:14:47.313995Z","iopub.status.idle":"2024-04-30T19:14:47.380284Z","shell.execute_reply.started":"2024-04-30T19:14:47.313962Z","shell.execute_reply":"2024-04-30T19:14:47.379328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional depth=1 files\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\n# test_debitcard_1 = pl.read_csv(pathway + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.382704Z","iopub.execute_input":"2024-04-30T19:14:47.382993Z","iopub.status.idle":"2024-04-30T19:14:47.395100Z","shell.execute_reply.started":"2024-04-30T19:14:47.382969Z","shell.execute_reply":"2024-04-30T19:14:47.394116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use aggregation functions in tables with depth >=1\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.396374Z","iopub.execute_input":"2024-04-30T19:14:47.396729Z","iopub.status.idle":"2024-04-30T19:14:47.406134Z","shell.execute_reply.started":"2024-04-30T19:14:47.396696Z","shell.execute_reply":"2024-04-30T19:14:47.404823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"), \n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.407553Z","iopub.execute_input":"2024-04-30T19:14:47.407990Z","iopub.status.idle":"2024-04-30T19:14:47.427002Z","shell.execute_reply.started":"2024-04-30T19:14:47.407945Z","shell.execute_reply":"2024-04-30T19:14:47.426102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.428252Z","iopub.execute_input":"2024-04-30T19:14:47.428604Z","iopub.status.idle":"2024-04-30T19:14:47.445895Z","shell.execute_reply.started":"2024-04-30T19:14:47.428571Z","shell.execute_reply":"2024-04-30T19:14:47.444900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n# Based on training preprocessing, should be 194 - 1 = 193 cols\ndrop_cols = ['clientscnt_136L', 'datelastinstal40dpd_247D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'interestrategrace_34L', 'isbidproductrequest_292L', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'payvacationpostpone_4187118D', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'riskassesment_302T', 'riskassesment_940T', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.447128Z","iopub.execute_input":"2024-04-30T19:14:47.447465Z","iopub.status.idle":"2024-04-30T19:14:47.485043Z","shell.execute_reply.started":"2024-04-30T19:14:47.447431Z","shell.execute_reply":"2024-04-30T19:14:47.484099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1,test_credit_bureau_b_1,test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats   \ngc.collect()  ","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.486187Z","iopub.execute_input":"2024-04-30T19:14:47.486447Z","iopub.status.idle":"2024-04-30T19:14:47.896663Z","shell.execute_reply.started":"2024-04-30T19:14:47.486424Z","shell.execute_reply":"2024-04-30T19:14:47.895450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Part 2","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.897961Z","iopub.execute_input":"2024-04-30T19:14:47.898316Z","iopub.status.idle":"2024-04-30T19:14:47.932464Z","shell.execute_reply.started":"2024-04-30T19:14:47.898288Z","shell.execute_reply":"2024-04-30T19:14:47.931672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.933463Z","iopub.execute_input":"2024-04-30T19:14:47.933743Z","iopub.status.idle":"2024-04-30T19:14:47.942290Z","shell.execute_reply.started":"2024-04-30T19:14:47.933719Z","shell.execute_reply":"2024-04-30T19:14:47.941320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    custom_range_agg(pl.col(\"annuity_853A\")).alias('annuity_853A_range'),\n    pl.col(\"byoccupationinc_3656910L\").sum().alias(\"byoccupationinc_3656910L_sum\"),\n    custom_range_agg(pl.col(\"byoccupationinc_3656910L\")).alias('byoccupationinc_3656910L_range'),\n    pl.col(\"childnum_21L\").sum().alias(\"childnum_21L_sum\"),\n    custom_range_agg(pl.col(\"childnum_21L\")).alias('childnum_21L_range'),\n    pl.col(\"credacc_credlmt_575A\").sum().alias(\"credacc_credlmt_575A_sum\"),\n    custom_range_agg(pl.col(\"credacc_credlmt_575A\")).alias('credacc_credlmt_575A_range'),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    custom_range_agg(pl.col(\"downpmt_134A\")).alias('downpmt_134A_range'),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    custom_range_agg(pl.col(\"mainoccupationinc_437A\")).alias('mainoccupationinc_437A_range'),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    custom_range_agg(pl.col(\"pmtnum_8L\")).alias('pmtnum_8L_range'),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"),\n    custom_range_agg(pl.col(\"tenor_203L\")).alias('tenor_203L_range'))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.943602Z","iopub.execute_input":"2024-04-30T19:14:47.944027Z","iopub.status.idle":"2024-04-30T19:14:47.968609Z","shell.execute_reply.started":"2024-04-30T19:14:47.943988Z","shell.execute_reply":"2024-04-30T19:14:47.967731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"),\n    custom_range_agg(pl.col(\"amount_4527230A\")).alias('amount_4527230A_range'))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.974717Z","iopub.execute_input":"2024-04-30T19:14:47.975012Z","iopub.status.idle":"2024-04-30T19:14:47.983926Z","shell.execute_reply.started":"2024-04-30T19:14:47.974987Z","shell.execute_reply":"2024-04-30T19:14:47.983088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = compare_cols(test_applprev_1_feats_1)\ntest_tax_registry_a_1_feats = compare_cols(test_tax_registry_a_1_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.985033Z","iopub.execute_input":"2024-04-30T19:14:47.985387Z","iopub.status.idle":"2024-04-30T19:14:47.998270Z","shell.execute_reply.started":"2024-04-30T19:14:47.985362Z","shell.execute_reply":"2024-04-30T19:14:47.997213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    custom_range_agg(pl.col(\"monthlyinstlamount_332A\")).alias('monthlyinstlamount_332A_range'),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    custom_range_agg(pl.col(\"monthlyinstlamount_674A\")).alias('monthlyinstlamount_674A_range'),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    custom_range_agg(pl.col(\"numberofinstls_229L\")).alias('numberofinstls_229L_range'),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    custom_range_agg(pl.col(\"numberofoverdueinstlmax_1151L\")).alias('numberofoverdueinstlmax_1151L_range'),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"),\n    custom_range_agg(pl.col(\"totalamount_6A\")).alias('totalamount_6A_range'))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))\n\ntest_credit_bureau_a_1_feats = compare_cols(test_credit_bureau_a_1_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:47.999917Z","iopub.execute_input":"2024-04-30T19:14:48.000491Z","iopub.status.idle":"2024-04-30T19:14:48.015784Z","shell.execute_reply.started":"2024-04-30T19:14:48.000437Z","shell.execute_reply":"2024-04-30T19:14:48.014873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','amount_4917619A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.016931Z","iopub.execute_input":"2024-04-30T19:14:48.017272Z","iopub.status.idle":"2024-04-30T19:14:48.037248Z","shell.execute_reply.started":"2024-04-30T19:14:48.017195Z","shell.execute_reply":"2024-04-30T19:14:48.036281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1,test_credit_bureau_a_1\ndel test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.038366Z","iopub.execute_input":"2024-04-30T19:14:48.038627Z","iopub.status.idle":"2024-04-30T19:14:48.440845Z","shell.execute_reply.started":"2024-04-30T19:14:48.038605Z","shell.execute_reply":"2024-04-30T19:14:48.439552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Part 3","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 2\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.442216Z","iopub.execute_input":"2024-04-30T19:14:48.442528Z","iopub.status.idle":"2024-04-30T19:14:48.452344Z","shell.execute_reply.started":"2024-04-30T19:14:48.442500Z","shell.execute_reply":"2024-04-30T19:14:48.451399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.453812Z","iopub.execute_input":"2024-04-30T19:14:48.454121Z","iopub.status.idle":"2024-04-30T19:14:48.479769Z","shell.execute_reply.started":"2024-04-30T19:14:48.454086Z","shell.execute_reply":"2024-04-30T19:14:48.479100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_1073P\")).alias('pmts_dpd_1073P_range'),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_303P\")).alias('pmts_dpd_303P_range'),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1140A\")).alias('pmts_overdue_1140A_range'),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1152A\")).alias('pmts_overdue_1152A_range'))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.480826Z","iopub.execute_input":"2024-04-30T19:14:48.481199Z","iopub.status.idle":"2024-04-30T19:14:48.491709Z","shell.execute_reply.started":"2024-04-30T19:14:48.481164Z","shell.execute_reply":"2024-04-30T19:14:48.490792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_2_feats = compare_cols(test_credit_bureau_a_2_feats)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.492895Z","iopub.execute_input":"2024-04-30T19:14:48.493280Z","iopub.status.idle":"2024-04-30T19:14:48.505728Z","shell.execute_reply.started":"2024-04-30T19:14:48.493248Z","shell.execute_reply":"2024-04-30T19:14:48.504766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.506918Z","iopub.execute_input":"2024-04-30T19:14:48.507239Z","iopub.status.idle":"2024-04-30T19:14:48.519862Z","shell.execute_reply.started":"2024-04-30T19:14:48.507213Z","shell.execute_reply":"2024-04-30T19:14:48.518988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2, test_person_2, test_credit_bureau_a_2\ndel test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.520999Z","iopub.execute_input":"2024-04-30T19:14:48.521312Z","iopub.status.idle":"2024-04-30T19:14:48.913690Z","shell.execute_reply.started":"2024-04-30T19:14:48.521287Z","shell.execute_reply":"2024-04-30T19:14:48.912566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n# Convert back to polars for datetime \njoin_test = pl.from_pandas(join_test)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.915318Z","iopub.execute_input":"2024-04-30T19:14:48.916015Z","iopub.status.idle":"2024-04-30T19:14:48.955291Z","shell.execute_reply.started":"2024-04-30T19:14:48.915976Z","shell.execute_reply":"2024-04-30T19:14:48.954307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_test.head() ","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.956623Z","iopub.execute_input":"2024-04-30T19:14:48.956962Z","iopub.status.idle":"2024-04-30T19:14:48.972471Z","shell.execute_reply.started":"2024-04-30T19:14:48.956933Z","shell.execute_reply":"2024-04-30T19:14:48.971417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Join Test Data","metadata":{}},{"cell_type":"code","source":"test = join_test.with_columns(pl.col('date_decision').cast(pl.Date))\n# Feature engineer date columns\ndate_list = ['datefirstoffer_1144D', 'datelastunpaid_3546854D', 'dtlastpmtallstes_4499206D', 'firstclxcampaign_1125D', 'lastdelinqdate_224D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'birthdate_574D', 'responsedate_1012D', 'responsedate_4527233D', 'responsedate_4917613D', 'empl_employedfrom_271D',\n            'firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D','birth_259D']\n\n# Calculates date_diff as date_decision minus each date column from above\nfor col in date_list:\n    test = test.with_columns(pl.col(col).cast(pl.Date))\n    test = test.with_columns(\n        ((pl.col(\"date_decision\") - pl.col(col)) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(f\"{col}_diff\"))\n\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\n# Convert _range features to Float64\nfor col in test.columns:\n        if col[-5:-1] in (\"range\"):\n            test = test.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            \ndrop_list = ['date_decision','MONTH'] + date_list\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(drop_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test, dfs\ngc.collect()\n\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:48.973673Z","iopub.execute_input":"2024-04-30T19:14:48.974005Z","iopub.status.idle":"2024-04-30T19:14:49.412229Z","shell.execute_reply.started":"2024-04-30T19:14:48.973978Z","shell.execute_reply":"2024-04-30T19:14:49.411230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noticed these numeric variables were parsed as strings, changed them all to float (same as training preprocessing)\nnum_list = ['numinstlswithdpd5_4187116L', 'numinstmatpaidtearly2d_4499204L','numinstpaid_4499208L','numinstpaidearly3dest_4493216L','numinstpaidearly5dest_4493211L',\n           'numinstpaidearly5dobd_4499205L','numinstpaidearlyest_4493214L','numinstpaidlastcontr_4325080L','numinstregularpaidest_4493210L','numinsttopaygrest_4493213L','numinstunpaidmaxest_4493212L',\n           'contractssum_5085716L','days120_123L','days180_256L','days360_512L','firstquarter_103L','fourthquarter_440L','numberofqueries_373L','pmtscount_423L','secondquarter_766L','thirdquarter_1082L',\n           'days30_165L','days90_310L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.413563Z","iopub.execute_input":"2024-04-30T19:14:49.413919Z","iopub.status.idle":"2024-04-30T19:14:49.430859Z","shell.execute_reply.started":"2024-04-30T19:14:49.413890Z","shell.execute_reply":"2024-04-30T19:14:49.429009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same as training preprocessing (features either all the same or categorical variables with unique categories > 150)\ndrop_list = ['commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L','addres_zip_823M',\n            'lastapprcommoditytypec_5251766M','lastrejectcommodtypec_5251769M','previouscontdistrict_112M', 'addres_district_368M']\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\n# Drop rows with all na\ntest.dropna(axis=1, how='all', inplace=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.432629Z","iopub.execute_input":"2024-04-30T19:14:49.432975Z","iopub.status.idle":"2024-04-30T19:14:49.470219Z","shell.execute_reply.started":"2024-04-30T19:14:49.432946Z","shell.execute_reply":"2024-04-30T19:14:49.469210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Imputation","metadata":{}},{"cell_type":"code","source":"\"\"\"\nprint(np.count_nonzero(test.isnull()))\n# Impute missing values if training data was also imputed\ntest = imputer(test)\nprint(np.count_nonzero(test.isnull()))\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.471519Z","iopub.execute_input":"2024-04-30T19:14:49.471833Z","iopub.status.idle":"2024-04-30T19:14:49.479345Z","shell.execute_reply.started":"2024-04-30T19:14:49.471804Z","shell.execute_reply":"2024-04-30T19:14:49.478207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See boolean columns\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.480556Z","iopub.execute_input":"2024-04-30T19:14:49.480843Z","iopub.status.idle":"2024-04-30T19:14:49.490790Z","shell.execute_reply.started":"2024-04-30T19:14:49.480819Z","shell.execute_reply":"2024-04-30T19:14:49.489764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify dtypes are correct\n#pd.set_option('display.max_rows', None)\n#test.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.492273Z","iopub.execute_input":"2024-04-30T19:14:49.493009Z","iopub.status.idle":"2024-04-30T19:14:49.501001Z","shell.execute_reply.started":"2024-04-30T19:14:49.492970Z","shell.execute_reply":"2024-04-30T19:14:49.499883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ncat_cols = test.select_dtypes(include=['category', 'object']).columns.tolist()\n# Create dummies for all cat columns if training data used dummies, not dropping first to keep column names same as training\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.502319Z","iopub.execute_input":"2024-04-30T19:14:49.502625Z","iopub.status.idle":"2024-04-30T19:14:49.512251Z","shell.execute_reply.started":"2024-04-30T19:14:49.502601Z","shell.execute_reply":"2024-04-30T19:14:49.511417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = reduce_mem_usage(test)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.513781Z","iopub.execute_input":"2024-04-30T19:14:49.514293Z","iopub.status.idle":"2024-04-30T19:14:49.633368Z","shell.execute_reply.started":"2024-04-30T19:14:49.514258Z","shell.execute_reply":"2024-04-30T19:14:49.632371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Train data","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv('/kaggle/input/training/train251_noimpute.csv').pipe(set_table_dtypes).pipe(convert_strings)\nfor col in train.columns:\n        if col[-5:-1] in (\"range\"):\n            train = train.with_columns(pl.col(col).cast(pl.Float64).alias(col))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:14:49.634778Z","iopub.execute_input":"2024-04-30T19:14:49.635602Z","iopub.status.idle":"2024-04-30T19:15:05.337889Z","shell.execute_reply.started":"2024-04-30T19:14:49.635564Z","shell.execute_reply":"2024-04-30T19:15:05.336939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert polars df to pandas so pandas specific methods/attributes work later, seems more memory efficient to load as pl and convert to pd than load as pd\ntrain = train.to_pandas()\n# Get list of ids for submission file\nids = test['case_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:15:05.339566Z","iopub.execute_input":"2024-04-30T19:15:05.340582Z","iopub.status.idle":"2024-04-30T19:15:09.356923Z","shell.execute_reply.started":"2024-04-30T19:15:05.340539Z","shell.execute_reply":"2024-04-30T19:15:09.355731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Make sure all cols are numeric if data was converted to dummies\nconvert_cols = train.select_dtypes(include=['category', 'object']).columns.tolist()\nfor col in convert_cols:\n    train[col]= pd.to_numeric(train[col])\ntrain.shape\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:15:09.358630Z","iopub.execute_input":"2024-04-30T19:15:09.359042Z","iopub.status.idle":"2024-04-30T19:15:09.366457Z","shell.execute_reply.started":"2024-04-30T19:15:09.359005Z","shell.execute_reply":"2024-04-30T19:15:09.365452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary Stats, Corr Heatmap, Additional feature analysis","metadata":{}},{"cell_type":"code","source":"# Summary Stats, features considered were taken from feature importance after running the model\npd.options.display.float_format = \"{:.2f}\".format\nimpt_list = ['incometype_1044T', 'lastapprcommoditycat_1041M', 'price_1097A', 'maritalst_385M', 'lastcancelreason_561M',\n            'pmts_overdue_1140A_sum', 'lastrejectdate_50D_diff', 'pctinstlsallpaidlate4d_3546849L', 'totalamount_6A_sum',\n            'pmtnum_254L', 'birth_259D_diff', 'avgdpdtolclosure24_3658938P', 'lastrejectcommoditycat_161M', 'education_1103M', 'sex_738L']\nimpt_num_list = ['price_1097A', 'pmts_overdue_1140A_sum', 'lastrejectdate_50D_diff', 'pctinstlsallpaidlate4d_3546849L', 'totalamount_6A_sum',\n                'pmtnum_254L', 'birth_259D_diff', 'avgdpdtolclosure24_3658938P']\nimpt_cat_list = [col for col in impt_list if col not in impt_num_list]\nimpt_list.sort()\ntrain[impt_list].describe()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:15:09.367711Z","iopub.execute_input":"2024-04-30T19:15:09.368014Z","iopub.status.idle":"2024-04-30T19:15:10.135522Z","shell.execute_reply.started":"2024-04-30T19:15:09.367985Z","shell.execute_reply":"2024-04-30T19:15:10.134554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Boxplots of numeric variables\nfig, axes = plt.subplots(nrows=1, ncols=8, figsize=(30, 5))\n\nfor i, feature in enumerate(impt_num_list):\n    sns.boxplot(x='target', y=feature, data=train, ax=axes[i])\n    axes[i].set_title(feature)\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:15:10.136862Z","iopub.execute_input":"2024-04-30T19:15:10.137282Z","iopub.status.idle":"2024-04-30T19:15:14.340180Z","shell.execute_reply.started":"2024-04-30T19:15:10.137247Z","shell.execute_reply":"2024-04-30T19:15:14.339112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Bar plots of mean default rate by group for incometype and sex\nplt.figure(figsize=(21,5))\nax = sns.barplot(x = 'incometype_1044T', y='target',data=train, errorbar=None)\nax.axhline(0.0314, color='red')\nax.bar_label(ax.containers[0], fmt='%.3f', label_type='center')\nplt.title(\"Percentage of Defaults Grouped by Income Type\")\nplt.ylabel('Default Rate')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:15:14.341368Z","iopub.execute_input":"2024-04-30T19:15:14.341659Z","iopub.status.idle":"2024-04-30T19:15:14.799376Z","shell.execute_reply.started":"2024-04-30T19:15:14.341634Z","shell.execute_reply":"2024-04-30T19:15:14.798462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(4,4))\nax = sns.barplot(x = 'sex_738L', y='target',data=train, errorbar=None)\nax.axhline(0.0314, color='red')\nax.bar_label(ax.containers[0], fmt='%.3f', label_type='center')\nplt.title(\"Percentage of Defaults Grouped by Gender\")\nplt.ylabel('Default Rate')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T19:15:14.806887Z","iopub.execute_input":"2024-04-30T19:15:14.807210Z","iopub.status.idle":"2024-04-30T19:15:15.106062Z","shell.execute_reply.started":"2024-04-30T19:15:14.807184Z","shell.execute_reply":"2024-04-30T19:15:15.105120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correlation heatmap of impt numeric features\nimpt_num_list.sort()\nmatrix = train[impt_num_list].corr().round(2)\nsns.heatmap(matrix, annot=True, vmax=1, vmin=-1, center=0, cmap='vlag')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:34:48.173113Z","iopub.execute_input":"2024-04-30T17:34:48.173394Z","iopub.status.idle":"2024-04-30T17:34:49.030352Z","shell.execute_reply.started":"2024-04-30T17:34:48.173370Z","shell.execute_reply":"2024-04-30T17:34:49.029331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impt numeric cols, mean based on target label\nfor col in impt_num_list:\n    print(train.groupby('target')[col].mean())","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:34:49.031734Z","iopub.execute_input":"2024-04-30T17:34:49.032110Z","iopub.status.idle":"2024-04-30T17:34:49.286852Z","shell.execute_reply.started":"2024-04-30T17:34:49.032070Z","shell.execute_reply":"2024-04-30T17:34:49.285896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Table of numeric means grouped by target label\nmean_df = pd.DataFrame({'price_1097A':[34297.68, 39603.99],'pmts_overdue_1140A_sum':[49231.74, 83557.00],'lastrejectdate_50D_diff':[846.42, 573.49],\n                       'pctinstlsallpaidlate4d_3546849L':[0.11, 0.24] , 'totalamount_6A_sum':[340655.96, 203042.52],'pmtnum_254L':[17.01, 18.95], \n                       'birth_259D_diff':[16326.43, 14734.13], 'avgdpdtolclosure24_3658938P':[42.76, 137.11]})\nmean_df = mean_df.reindex(sorted(mean_df.columns), axis=1)\nmean_df","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:34:49.288108Z","iopub.execute_input":"2024-04-30T17:34:49.288411Z","iopub.status.idle":"2024-04-30T17:34:49.304803Z","shell.execute_reply.started":"2024-04-30T17:34:49.288384Z","shell.execute_reply":"2024-04-30T17:34:49.303666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate P-values for one-sided Z tests of means \nimpt_num_list.sort()\nfor col in impt_num_list:\n    temp = train[train[[col]].notna().any(axis=1)]\n    temp1 = temp.loc[(temp['target'] ==0)][col].values\n    temp2 = temp.loc[(temp['target'] ==1)][col].values\n    # Used both larger and smaller one-sided tests\n    z, p = ztest(temp1, temp2, value=0, alternative='larger', usevar='unequal')\n    z1, p1 = ztest(temp1, temp2, value=0, alternative='smaller', usevar='unequal')\n    print(f'P-Value for {col}: {min(p,p1)}') # Not the most technically correct implementation, but output was verified that the correct p-value is the min","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:34:49.306315Z","iopub.execute_input":"2024-04-30T17:34:49.306600Z","iopub.status.idle":"2024-04-30T17:35:04.540185Z","shell.execute_reply.started":"2024-04-30T17:34:49.306576Z","shell.execute_reply":"2024-04-30T17:35:04.539180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tables of mean target for each group in impt cat variables and their count\npd.options.display.float_format = \"{:.4f}\".format\npd.set_option('display.max_rows', 20)\nimpt_cat_list.sort()\n# display df showing target percentage and count for each impt cat feature\nfor col in impt_cat_list:\n    a = train.groupby(col, observed=True)['target'].sum()\n    b = train.groupby(col, observed=True)['target'].count()\n    temp = pd.DataFrame(a/b)\n    temp.insert(1, \"count\", b.values, True)\n    display(temp.sort_values('target',ascending=False).head(20))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:04.541615Z","iopub.execute_input":"2024-04-30T17:35:04.541989Z","iopub.status.idle":"2024-04-30T17:35:04.963569Z","shell.execute_reply.started":"2024-04-30T17:35:04.541955Z","shell.execute_reply":"2024-04-30T17:35:04.962687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Additional Preprocessing","metadata":{}},{"cell_type":"code","source":"train = reduce_mem_usage(train)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:04.964962Z","iopub.execute_input":"2024-04-30T17:35:04.965434Z","iopub.status.idle":"2024-04-30T17:35:08.766998Z","shell.execute_reply.started":"2024-04-30T17:35:04.965398Z","shell.execute_reply":"2024-04-30T17:35:08.765954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.set_option('display.max_rows', None)\n#train.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:08.768228Z","iopub.execute_input":"2024-04-30T17:35:08.770180Z","iopub.status.idle":"2024-04-30T17:35:08.773862Z","shell.execute_reply.started":"2024-04-30T17:35:08.770142Z","shell.execute_reply":"2024-04-30T17:35:08.772868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only select common columns to use\ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest=test[common_columns]\n\n# Subset train with only columns seen in test + target\ntrain = train[common_columns+['target']]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:08.774903Z","iopub.execute_input":"2024-04-30T17:35:08.775148Z","iopub.status.idle":"2024-04-30T17:35:10.301439Z","shell.execute_reply.started":"2024-04-30T17:35:08.775126Z","shell.execute_reply":"2024-04-30T17:35:10.300581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Fit on stratified sample\n# Used for testing code\ntrain_sample = train.groupby('target', group_keys=False).apply(lambda x: x.sample(frac=0.01)).reset_index(drop=True)\ntrain_sample.head()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:10.302643Z","iopub.execute_input":"2024-04-30T17:35:10.302929Z","iopub.status.idle":"2024-04-30T17:35:11.722442Z","shell.execute_reply.started":"2024-04-30T17:35:10.302905Z","shell.execute_reply":"2024-04-30T17:35:11.721579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.loc[:,'target'].to_frame('target')\nX = train.drop(['target'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:11.723692Z","iopub.execute_input":"2024-04-30T17:35:11.724068Z","iopub.status.idle":"2024-04-30T17:35:11.733799Z","shell.execute_reply.started":"2024-04-30T17:35:11.724032Z","shell.execute_reply":"2024-04-30T17:35:11.732767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check target distribution is same after all the preprocessing/sampling\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(),4))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:11.734917Z","iopub.execute_input":"2024-04-30T17:35:11.735169Z","iopub.status.idle":"2024-04-30T17:35:11.747473Z","shell.execute_reply.started":"2024-04-30T17:35:11.735146Z","shell.execute_reply":"2024-04-30T17:35:11.746522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:11.748754Z","iopub.execute_input":"2024-04-30T17:35:11.749105Z","iopub.status.idle":"2024-04-30T17:35:11.907794Z","shell.execute_reply.started":"2024-04-30T17:35:11.749075Z","shell.execute_reply":"2024-04-30T17:35:11.906930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do not include case_id, or week_num as numeric \nnumeric_cols = test.select_dtypes(include=['number']).columns.tolist()\nnumeric_cols.remove('case_id')\nnumeric_cols.remove('WEEK_NUM')\n#print(numeric_cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:11.909015Z","iopub.execute_input":"2024-04-30T17:35:11.909344Z","iopub.status.idle":"2024-04-30T17:35:11.922493Z","shell.execute_reply.started":"2024-04-30T17:35:11.909312Z","shell.execute_reply":"2024-04-30T17:35:11.921768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standardize data using minmaxscaler\nwarnings.filterwarnings(\"ignore\")\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nwarnings.filterwarnings(\"default\")\n\nX.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:11.923651Z","iopub.execute_input":"2024-04-30T17:35:11.924442Z","iopub.status.idle":"2024-04-30T17:35:12.868518Z","shell.execute_reply.started":"2024-04-30T17:35:11.924367Z","shell.execute_reply":"2024-04-30T17:35:12.867656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"code","source":"# Drop case_id and week_num from features\nweeks = X[\"WEEK_NUM\"]\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n# Sort columns in alphabetical order for training so columns match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\n\nprint(X_feats.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:12.869725Z","iopub.execute_input":"2024-04-30T17:35:12.870006Z","iopub.status.idle":"2024-04-30T17:35:12.908255Z","shell.execute_reply.started":"2024-04-30T17:35:12.869983Z","shell.execute_reply":"2024-04-30T17:35:12.907405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:12.909332Z","iopub.execute_input":"2024-04-30T17:35:12.909592Z","iopub.status.idle":"2024-04-30T17:35:13.035865Z","shell.execute_reply.started":"2024-04-30T17:35:12.909570Z","shell.execute_reply":"2024-04-30T17:35:13.035025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include=['category', 'object']).columns.tolist()\n#print(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:13.036939Z","iopub.execute_input":"2024-04-30T17:35:13.037197Z","iopub.status.idle":"2024-04-30T17:35:13.047775Z","shell.execute_reply.started":"2024-04-30T17:35:13.037174Z","shell.execute_reply":"2024-04-30T17:35:13.046861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_feats[cat_cols] = X_feats[cat_cols].astype(str)\ntest[cat_cols] = test[cat_cols].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:13.048956Z","iopub.execute_input":"2024-04-30T17:35:13.049366Z","iopub.status.idle":"2024-04-30T17:35:13.115976Z","shell.execute_reply.started":"2024-04-30T17:35:13.049335Z","shell.execute_reply":"2024-04-30T17:35:13.115276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nfitted_models_cat = []\nfitted_models_lgb = []\n#fitted_models_xgb = []\n\ncv_scores_cat = []\ncv_scores_lgb = []\n#cv_scores_xgb = []\n\n# Note: Currently running with GPU, change params if no accelerator\n# Testing ensemble with XGB yielded slightly worse results, so left it out of model\ngrid_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 15,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2500,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"random_state\": 3107,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"verbose\": -1,\n    'device':'gpu',\n    #'device': 'cpu'\n}\n\n\"\"\"\ngrid_params2 = {\n    \"booster\": \"gbtree\",\n    \"objective\": \"binary:logistic\",\n    \"eval_metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"alpha\": 0.1,  \n    \"lambda\": 10,  \n    \"tree_method\": 'gpu_hist',\n    #\"tree_method\": 'auto',\n    \"random_state\": 123,\n    \"verbosity\": 0,\n    \"enable_categorical\":True,\n}\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:13.116926Z","iopub.execute_input":"2024-04-30T17:35:13.117182Z","iopub.status.idle":"2024-04-30T17:35:13.125748Z","shell.execute_reply.started":"2024-04-30T17:35:13.117160Z","shell.execute_reply":"2024-04-30T17:35:13.124923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \nwarnings.filterwarnings(\"ignore\")\n\n# Ensemble model inspired by other public competition notebooks\n# Group by weeks to account for gini instability weekly metric\nfor idx_train, idx_valid in cv.split(X_feats, y, groups=weeks): \n    # Split training in every fold by setting the split indexes\n    X_train, y_train = X_feats.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X_feats.iloc[idx_valid], y.iloc[idx_valid]\n    # Pool used for CatBoost\n    train_pool = Pool(X_train, y_train,cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid,cat_features=cat_cols)\n    \n    clf = CatBoostClassifier(\n    eval_metric='AUC', task_type='GPU', # Change if not using GPU\n    learning_rate=0.05, iterations = 5000, # default 1k, lower if needed\n    random_seed=3107)  \n    clf.fit(train_pool, eval_set=val_pool,verbose=300)\n    # Append model and auc_score to their appropriate list\n    fitted_models_cat.append(clf)\n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n    \n    X_train[cat_cols] = X_train[cat_cols].astype(\"category\")\n    X_valid[cat_cols] = X_valid[cat_cols].astype(\"category\")\n    \n    # Train lgbm model, stopping if no improvement in auc_score in 200 rounds\n    model = lgb.LGBMClassifier(**grid_params)\n    model.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(200)] )\n    # Again, append the model and auc_score\n    fitted_models_lgb.append(model)\n    y_pred_valid = model.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_lgb.append(auc_score)\n    \n    \"\"\"\n    # Final XGB model with same early stopping \n    model2 = xgb.XGBClassifier(**grid_params2)\n    model2.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        early_stopping_rounds=100, verbose=False)\n    # Again, append model and auc_score\n    fitted_models_xgb.append(model2)\n    y_pred_valid = model2.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_xgb.append(auc_score)\n    \"\"\"\n    \n    del clf, model\n    gc.collect()\n    \nprint(\"CV CAT AUC scores: \", cv_scores_cat)\nprint(\"Maximum CV CAT AUC score: \", max(cv_scores_cat))\n\nprint(\"CV LGB AUC scores: \", cv_scores_lgb)\nprint(\"Maximum CV LGB AUC score: \", max(cv_scores_lgb))\n\n\"\"\"\nprint(\"CV XGB AUC scores: \", cv_scores_xgb)\nprint(\"Maximum CV XGB AUC score: \", max(cv_scores_xgb))\n\"\"\"\n\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:35:13.126994Z","iopub.execute_input":"2024-04-30T17:35:13.127641Z","iopub.status.idle":"2024-04-30T17:36:01.106736Z","shell.execute_reply.started":"2024-04-30T17:35:13.127599Z","shell.execute_reply":"2024-04-30T17:36:01.105767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the best cv model, inspired by other competition notebooks\nclass VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators[:5]]    \n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        y_preds += [estimator.predict(X) for estimator in self.estimators[5:10]]\n        \n        # y_preds += [estimator.predict(X) for estimator in self.estimators[10:]]\n        return np.mean(y_preds, axis=0)\n    \n    # main use case for this class \n    def predict_proba(self, X):\n        # predict_proba for CatBoost\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]    \n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        # predict_proba for LGBM\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[5:10]]\n        \n        # predict_proba for XGB\n        # y_preds += [estimator.predict_proba(X) for estimator in self.estimators[10:]]\n        return np.mean(y_preds, axis=0)\n    \n# Create VotingModel object\nmodel = VotingModel(fitted_models_cat + fitted_models_lgb)\n\n# Extract best model for each classifier for feature importance plotting\nmodel_lgb = fitted_models_lgb[np.argmax(cv_scores_lgb)]\nmodel_cat = fitted_models_cat[np.argmax(cv_scores_cat)]\n# model_xgb = fitted_models_xgb[np.argmax(cv_scores_xgb)]","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:01.108017Z","iopub.execute_input":"2024-04-30T17:36:01.108296Z","iopub.status.idle":"2024-04-30T17:36:01.117962Z","shell.execute_reply.started":"2024-04-30T17:36:01.108272Z","shell.execute_reply":"2024-04-30T17:36:01.116986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del fitted_models_cat, fitted_models_lgb\ngc.collect() ","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:01.119146Z","iopub.execute_input":"2024-04-30T17:36:01.119454Z","iopub.status.idle":"2024-04-30T17:36:01.235434Z","shell.execute_reply.started":"2024-04-30T17:36:01.119425Z","shell.execute_reply":"2024-04-30T17:36:01.234551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion Matrix and Classification Report","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\ny_pred = model.predict(X_feats)\ny_pred1 = [1 if num > 0.50 else 0 for num in y_pred]\n\ntarget_names= ['No Default', 'Default']\nmatrix = confusion_matrix(y, y_pred1, normalize='true')\ncm_display = ConfusionMatrixDisplay(confusion_matrix= matrix, display_labels=target_names).plot()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:01.236465Z","iopub.execute_input":"2024-04-30T17:36:01.236745Z","iopub.status.idle":"2024-04-30T17:36:03.500803Z","shell.execute_reply.started":"2024-04-30T17:36:01.236721Z","shell.execute_reply":"2024-04-30T17:36:03.499890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print out the report\nprint(classification_report(y, y_pred1, target_names = target_names))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:03.502067Z","iopub.execute_input":"2024-04-30T17:36:03.502357Z","iopub.status.idle":"2024-04-30T17:36:03.548894Z","shell.execute_reply.started":"2024-04-30T17:36:03.502332Z","shell.execute_reply":"2024-04-30T17:36:03.547869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot ROC AUC curve for validation data\n# I prefer skplt.metrics.plot_roc_curve but there is an error with the scipy version and would necessitate creating a custom environment.\n\nfpr, tpr, threshold = metrics.roc_curve(y_valid, y_pred_valid)\ndisplay = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=auc_score)\ndisplay.plot()\nplt.plot([0, 1], [0, 1],'r--')\nplt.title('ROC-AUC Curve for Validation Data')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:03.550165Z","iopub.execute_input":"2024-04-30T17:36:03.550473Z","iopub.status.idle":"2024-04-30T17:36:03.823965Z","shell.execute_reply.started":"2024-04-30T17:36:03.550433Z","shell.execute_reply":"2024-04-30T17:36:03.823056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_pred, y_pred1\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:03.825247Z","iopub.execute_input":"2024-04-30T17:36:03.825531Z","iopub.status.idle":"2024-04-30T17:36:03.958573Z","shell.execute_reply.started":"2024-04-30T17:36:03.825506Z","shell.execute_reply":"2024-04-30T17:36:03.957646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"code","source":"# For lgb model, ranks features based on number of splits\nlgb.plot_importance(model_lgb, figsize=(20,20), max_num_features = 20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:03.959725Z","iopub.execute_input":"2024-04-30T17:36:03.959994Z","iopub.status.idle":"2024-04-30T17:36:04.540083Z","shell.execute_reply.started":"2024-04-30T17:36:03.959970Z","shell.execute_reply":"2024-04-30T17:36:04.539157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb.plot_importance(model_lgb, figsize=(20,20), importance_type='gain', max_num_features =20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:04.541208Z","iopub.execute_input":"2024-04-30T17:36:04.541491Z","iopub.status.idle":"2024-04-30T17:36:05.116104Z","shell.execute_reply.started":"2024-04-30T17:36:04.541467Z","shell.execute_reply":"2024-04-30T17:36:05.115208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_feature_importance(importance,columns,model_name):\n    # Create arrays for feature importance and feature names\n    feature_importance = np.array(importance)\n    feature_names = np.array(columns)\n    \n    # Create a df using a dictionary\n    data={'feature_names':feature_names,'feature_importance':feature_importance}\n    temp_df = pd.DataFrame(data)\n    \n    #Sort the df in order of decreasing feature importance\n    temp_df.sort_values(by=['feature_importance'], ascending=False,inplace=True)\n    \n    plt.figure(figsize=(20,20))\n    # Plot only top 20 features \n    sns.barplot(x=temp_df['feature_importance'][:20], y=temp_df['feature_names'][:20])\n\n    plt.title(model_name)\n    plt.xlabel('Feature Importance')\n    plt.ylabel('Feature Names')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:05.117285Z","iopub.execute_input":"2024-04-30T17:36:05.117572Z","iopub.status.idle":"2024-04-30T17:36:05.124486Z","shell.execute_reply.started":"2024-04-30T17:36:05.117546Z","shell.execute_reply":"2024-04-30T17:36:05.123543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculates the importance of each feature by comparing the model’s performance with and without that feature. \n# Difference in the loss function obtained when including the feature versus excluding it from the model. The greater the difference, the more important the feature.\nplot_feature_importance(model_cat.get_feature_importance(type='PredictionValuesChange'), X_train.columns, \"CATBOOST\")\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:05.125798Z","iopub.execute_input":"2024-04-30T17:36:05.126352Z","iopub.status.idle":"2024-04-30T17:36:05.779741Z","shell.execute_reply.started":"2024-04-30T17:36:05.126320Z","shell.execute_reply":"2024-04-30T17:36:05.778828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Gain is derived from a complicated formula but can be summarized as the relative contribution of each feature to the model.\n# Total gain is just the sum of gains across all splits.\n# https://xgboost.readthedocs.io/en/latest/tutorials/model.html\nxgb_impt = model_xgb.get_booster().get_score(importance_type='total_gain')\nkeys = list(xgb_impt.keys())\nvalues = list(xgb_impt.values())\n\nplot_feature_importance(values, keys, \"XGB Total Gain\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:05.781065Z","iopub.execute_input":"2024-04-30T17:36:05.781454Z","iopub.status.idle":"2024-04-30T17:36:05.788515Z","shell.execute_reply.started":"2024-04-30T17:36:05.781419Z","shell.execute_reply":"2024-04-30T17:36:05.787572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nxgb_impt = model_xgb.get_booster().get_score(importance_type='gain')\nkeys = list(xgb_impt.keys())\nvalues = list(xgb_impt.values())\n\nplot_feature_importance(values, keys, \"XGB Gain\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:05.789539Z","iopub.execute_input":"2024-04-30T17:36:05.789824Z","iopub.status.idle":"2024-04-30T17:36:05.804489Z","shell.execute_reply.started":"2024-04-30T17:36:05.789800Z","shell.execute_reply":"2024-04-30T17:36:05.803591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_feats, X_train, X_valid, y_train, y_valid, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:05.807188Z","iopub.execute_input":"2024-04-30T17:36:05.807485Z","iopub.status.idle":"2024-04-30T17:36:05.954540Z","shell.execute_reply.started":"2024-04-30T17:36:05.807462Z","shell.execute_reply":"2024-04-30T17:36:05.953551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# Sort columns alphabetically to match loaded model\ntest = test.reindex(sorted(test.columns), axis=1)\n\npredictions = model.predict_proba(test.drop(['case_id', 'WEEK_NUM'], axis=1))","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:05.955772Z","iopub.execute_input":"2024-04-30T17:36:05.956181Z","iopub.status.idle":"2024-04-30T17:36:06.171543Z","shell.execute_reply.started":"2024-04-30T17:36:05.956157Z","shell.execute_reply":"2024-04-30T17:36:06.170631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(predictions)","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:06.172707Z","iopub.execute_input":"2024-04-30T17:36:06.172975Z","iopub.status.idle":"2024-04-30T17:36:06.178312Z","shell.execute_reply.started":"2024-04-30T17:36:06.172952Z","shell.execute_reply":"2024-04-30T17:36:06.177231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:06.179517Z","iopub.execute_input":"2024-04-30T17:36:06.180339Z","iopub.status.idle":"2024-04-30T17:36:06.315576Z","shell.execute_reply.started":"2024-04-30T17:36:06.180314Z","shell.execute_reply":"2024-04-30T17:36:06.314636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id': ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:06.316802Z","iopub.execute_input":"2024-04-30T17:36:06.317140Z","iopub.status.idle":"2024-04-30T17:36:06.330042Z","shell.execute_reply.started":"2024-04-30T17:36:06.317108Z","shell.execute_reply":"2024-04-30T17:36:06.329178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Final submission csv\nsubmission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-30T17:36:06.331415Z","iopub.execute_input":"2024-04-30T17:36:06.331773Z","iopub.status.idle":"2024-04-30T17:36:06.341816Z","shell.execute_reply.started":"2024-04-30T17:36:06.331742Z","shell.execute_reply":"2024-04-30T17:36:06.340976Z"},"trusted":true},"execution_count":null,"outputs":[]}]}