{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7870556,"sourceType":"datasetVersion","datasetId":4618198},{"sourceId":7941325,"sourceType":"datasetVersion","datasetId":4506020}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nfrom pprint import pprint\nimport scikitplot as skplt\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, confusion_matrix, ConfusionMatrixDisplay, classification_report, roc_auc_score\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, StratifiedGroupKFold\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import SVC\n\nfrom contextlib import suppress\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\nfrom pprint import pprint\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:31.902896Z","iopub.execute_input":"2024-04-04T05:12:31.903956Z","iopub.status.idle":"2024-04-04T05:12:33.919180Z","shell.execute_reply.started":"2024-04-04T05:12:31.903887Z","shell.execute_reply":"2024-04-04T05:12:33.918175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now I will define my User Defined Functions","metadata":{}},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\n#this function takes a data frame as input, and iterates over all the columns\n#each column ending with a P or an A, it casts the column to Float64\ndef set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated columns as Float64, tried combining sum and max, but did not work correctly\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\n#this function takes a data frame as input, and iterates over all the columns\n#each columns that is a string or an object...it converts the col to string type, then to cat type\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n#This function prints the percent of missing values\n# Changed this function to work for Pandas\ndef missing_values(df, threshold = 0.0):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\n# mode_without_nan represents the most frequent value in the column, exluding any missing values.\n# dropna() removes missing values NAN from the column, and returns a new series with non-null values\n# .mode() calculates the mode in the series\n# .values[0] extracts the first (or only) mode from .mode(), which is a series (we are accessing first element)\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == 'float64':\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:36.534229Z","iopub.execute_input":"2024-04-04T05:12:36.534860Z","iopub.status.idle":"2024-04-04T05:12:36.650950Z","shell.execute_reply.started":"2024-04-04T05:12:36.534825Z","shell.execute_reply":"2024-04-04T05:12:36.649836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Join the test data","metadata":{}},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway +  \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:39.602461Z","iopub.execute_input":"2024-04-04T05:12:39.603159Z","iopub.status.idle":"2024-04-04T05:12:39.649112Z","shell.execute_reply.started":"2024-04-04T05:12:39.603119Z","shell.execute_reply":"2024-04-04T05:12:39.648044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional depth=1 files\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\n# test_debitcard_1 = pl.read_csv(pathway + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:41.957298Z","iopub.execute_input":"2024-04-04T05:12:41.958089Z","iopub.status.idle":"2024-04-04T05:12:41.969746Z","shell.execute_reply.started":"2024-04-04T05:12:41.958053Z","shell.execute_reply":"2024-04-04T05:12:41.968775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use aggregation functions in tables with depth >=1\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:44.647639Z","iopub.execute_input":"2024-04-04T05:12:44.648533Z","iopub.status.idle":"2024-04-04T05:12:44.658272Z","shell.execute_reply.started":"2024-04-04T05:12:44.648494Z","shell.execute_reply":"2024-04-04T05:12:44.657239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"),\n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:49.034797Z","iopub.execute_input":"2024-04-04T05:12:49.035571Z","iopub.status.idle":"2024-04-04T05:12:49.049147Z","shell.execute_reply.started":"2024-04-04T05:12:49.035536Z","shell.execute_reply":"2024-04-04T05:12:49.047705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:51.499617Z","iopub.execute_input":"2024-04-04T05:12:51.500030Z","iopub.status.idle":"2024-04-04T05:12:51.517476Z","shell.execute_reply.started":"2024-04-04T05:12:51.499999Z","shell.execute_reply":"2024-04-04T05:12:51.515996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n\n# drop_cols = list_missing(test)\ndrop_cols = ['avgdbddpdlast3m_4187120P', 'avgdbdtollast24m_4525197P', 'avglnamtstart24m_4525187A', 'avgpmtlast12m_4525200A', 'bankacctype_710L', 'cardtype_51L', 'clientscnt_136L', 'datelastinstal40dpd_247D', 'dtlastpmtallstes_4499206D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'inittransactionamount_650A', 'interestrategrace_34L', 'isbidproductrequest_292L', 'isdebitcard_729L', 'lastdelinqdate_224D', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'maxdbddpdlast1m_3658939P', 'maxlnamtstart6m_4525199A', 'maxpmtlast3m_4525190A', 'mindbdtollast24m_4525191P', 'payvacationpostpone_4187118D', 'totinstallast1m_4525188A', 'typesuite_864L', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'birthdate_574D', 'contractssum_5085716L', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'pmtscount_423L', 'pmtssum_45A', 'responsedate_4917613D', 'riskassesment_302T', 'riskassesment_940T', 'empl_employedfrom_271D', 'empl_industry_691L', 'empl_employedtotal_800L', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\n# print(drop_cols)\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:54.098995Z","iopub.execute_input":"2024-04-04T05:12:54.099472Z","iopub.status.idle":"2024-04-04T05:12:54.126542Z","shell.execute_reply.started":"2024-04-04T05:12:54.099438Z","shell.execute_reply":"2024-04-04T05:12:54.125734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1,test_credit_bureau_b_1,test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats   \ngc.collect()  ","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:12:57.346386Z","iopub.execute_input":"2024-04-04T05:12:57.346822Z","iopub.status.idle":"2024-04-04T05:12:57.462370Z","shell.execute_reply.started":"2024-04-04T05:12:57.346792Z","shell.execute_reply":"2024-04-04T05:12:57.461031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 2","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:03.708289Z","iopub.execute_input":"2024-04-04T05:13:03.708748Z","iopub.status.idle":"2024-04-04T05:13:03.745888Z","shell.execute_reply.started":"2024-04-04T05:13:03.708717Z","shell.execute_reply":"2024-04-04T05:13:03.744773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:05.962617Z","iopub.execute_input":"2024-04-04T05:13:05.963548Z","iopub.status.idle":"2024-04-04T05:13:05.972791Z","shell.execute_reply.started":"2024-04-04T05:13:05.963512Z","shell.execute_reply":"2024-04-04T05:13:05.971722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    pl.col(\"byoccupationinc_3656910L\").max().alias(\"byoccupationinc_3656910L_max\"),\n    pl.col(\"childnum_21L\").max().alias(\"childnum_21L_max\"),\n    pl.col(\"credacc_credlmt_575A\").max().alias(\"credacc_credlmt_575A_max\"),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"))\n\ntest_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))\n\ntest_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:07.974649Z","iopub.execute_input":"2024-04-04T05:13:07.975114Z","iopub.status.idle":"2024-04-04T05:13:07.995457Z","shell.execute_reply.started":"2024-04-04T05:13:07.975079Z","shell.execute_reply":"2024-04-04T05:13:07.993751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1,test_credit_bureau_a_1\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:11.104564Z","iopub.execute_input":"2024-04-04T05:13:11.104993Z","iopub.status.idle":"2024-04-04T05:13:11.224898Z","shell.execute_reply.started":"2024-04-04T05:13:11.104961Z","shell.execute_reply":"2024-04-04T05:13:11.223700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','byoccupationinc_3656910L_max','familystate_726L', 'amount_4527230A_sum', 'amount_4917619A_sum', 'pmtamount_36A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:14.256652Z","iopub.execute_input":"2024-04-04T05:13:14.257142Z","iopub.status.idle":"2024-04-04T05:13:14.276583Z","shell.execute_reply.started":"2024-04-04T05:13:14.257108Z","shell.execute_reply":"2024-04-04T05:13:14.275418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:17.226426Z","iopub.execute_input":"2024-04-04T05:13:17.226812Z","iopub.status.idle":"2024-04-04T05:13:17.346853Z","shell.execute_reply.started":"2024-04-04T05:13:17.226785Z","shell.execute_reply":"2024-04-04T05:13:17.345943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 3","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 2\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\n\nsel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:19.831635Z","iopub.execute_input":"2024-04-04T05:13:19.832029Z","iopub.status.idle":"2024-04-04T05:13:19.865791Z","shell.execute_reply.started":"2024-04-04T05:13:19.831993Z","shell.execute_reply":"2024-04-04T05:13:19.864932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:22.026932Z","iopub.execute_input":"2024-04-04T05:13:22.027337Z","iopub.status.idle":"2024-04-04T05:13:22.037577Z","shell.execute_reply.started":"2024-04-04T05:13:22.027308Z","shell.execute_reply":"2024-04-04T05:13:22.036544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2, test_person_2, test_credit_bureau_a_2\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:24.503822Z","iopub.execute_input":"2024-04-04T05:13:24.504241Z","iopub.status.idle":"2024-04-04T05:13:24.622891Z","shell.execute_reply.started":"2024-04-04T05:13:24.504209Z","shell.execute_reply":"2024-04-04T05:13:24.621429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\").join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:27.857154Z","iopub.execute_input":"2024-04-04T05:13:27.858044Z","iopub.status.idle":"2024-04-04T05:13:27.867261Z","shell.execute_reply.started":"2024-04-04T05:13:27.858009Z","shell.execute_reply":"2024-04-04T05:13:27.866407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####join_data3.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n\n# Convert back to polars for date extraction \njoin_test = pl.from_pandas(join_test)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:35.466724Z","iopub.execute_input":"2024-04-04T05:13:35.467120Z","iopub.status.idle":"2024-04-04T05:13:35.505055Z","shell.execute_reply.started":"2024-04-04T05:13:35.467093Z","shell.execute_reply":"2024-04-04T05:13:35.503789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Join of the Test Data","metadata":{}},{"cell_type":"code","source":"join_test = join_test.with_columns(pl.col('date_decision','birth_259D').cast(pl.Date))\n#Feature engineer days diff between decision and birthdate\n\ntest = join_test.with_columns(\n    ((pl.col(\"date_decision\") - pl.col(\"birth_259D\")) / (24*60*60*1000)).cast(pl.Float64).alias(\"date_diff\"))\n\n\n\n\n# Drop uneeded date columns plus other columns\ndate_list = ['date_decision','MONTH','firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D', 'firstclxcampaign_1125D', \n'birth_259D','datefirstoffer_1144D', 'datelastunpaid_3546854D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'responsedate_1012D', 'responsedate_4527233D', 'requesttype_4525192L']\n\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(date_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test\ngc.collect()\n\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:37.748162Z","iopub.execute_input":"2024-04-04T05:13:37.748618Z","iopub.status.idle":"2024-04-04T05:13:37.901042Z","shell.execute_reply.started":"2024-04-04T05:13:37.748564Z","shell.execute_reply":"2024-04-04T05:13:37.899952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noticed some of these numeric variables were parsed as strings, changed them all to int \nnum_list = ['days120_123L','days180_256L','days30_165L','days360_512L','days90_310L','firstquarter_103L','numinstpaidlastcontr_4325080L',\n            'fourthquarter_440L','numinstlswithdpd5_4187116L','numberofqueries_373L', 'secondquarter_766L','thirdquarter_1082L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')\n\n# Drop some unneeded columns, based on training missing values, 0 range for numeric, or one unique category\ndrop_list = ['lastapprcommoditytypec_5251766M', 'lastrejectcommodtypec_5251769M','lastrejectcommoditycat_161M','lastrejectreasonclient_4145040M',\n            'previouscontdistrict_112M','lastapprcommoditycat_1041M', 'lastcancelreason_561M','lastrejectreason_759M', 'addres_zip_823M', 'addres_district_368M',\n            'commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L','paytype1st_925L_OTHER','paytype_783L_OTHER','applicationcnt_361L','empls_employer_name_740M_a55475b1']\n\n\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:41.017473Z","iopub.execute_input":"2024-04-04T05:13:41.018402Z","iopub.status.idle":"2024-04-04T05:13:41.072459Z","shell.execute_reply.started":"2024-04-04T05:13:41.018344Z","shell.execute_reply":"2024-04-04T05:13:41.071284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imputation","metadata":{}},{"cell_type":"code","source":"print(np.count_nonzero(test.isnull()))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:44.791175Z","iopub.execute_input":"2024-04-04T05:13:44.792255Z","iopub.status.idle":"2024-04-04T05:13:44.798993Z","shell.execute_reply.started":"2024-04-04T05:13:44.792219Z","shell.execute_reply":"2024-04-04T05:13:44.797868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = imputer(test)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:47.281231Z","iopub.execute_input":"2024-04-04T05:13:47.281683Z","iopub.status.idle":"2024-04-04T05:13:47.389668Z","shell.execute_reply.started":"2024-04-04T05:13:47.281651Z","shell.execute_reply":"2024-04-04T05:13:47.388368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.count_nonzero(test.isnull()))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:50.650795Z","iopub.execute_input":"2024-04-04T05:13:50.651302Z","iopub.status.idle":"2024-04-04T05:13:50.661378Z","shell.execute_reply.started":"2024-04-04T05:13:50.651269Z","shell.execute_reply":"2024-04-04T05:13:50.660234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I can now see that I reduced the missing value count from 457 to 0 after the imputation.","metadata":{}},{"cell_type":"code","source":"# See boolean columns\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:54.101564Z","iopub.execute_input":"2024-04-04T05:13:54.101973Z","iopub.status.idle":"2024-04-04T05:13:54.111626Z","shell.execute_reply.started":"2024-04-04T05:13:54.101943Z","shell.execute_reply":"2024-04-04T05:13:54.110503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include=['category']).columns.tolist()\n#create dummies for all cat columns, not dropping first to keep column names the same as training\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:13:57.572433Z","iopub.execute_input":"2024-04-04T05:13:57.573111Z","iopub.status.idle":"2024-04-04T05:13:57.639762Z","shell.execute_reply.started":"2024-04-04T05:13:57.573077Z","shell.execute_reply":"2024-04-04T05:13:57.638719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Model","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv('/kaggle/input/training/train_final_dummy.csv')\ntrain.head(20)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:14:02.333911Z","iopub.execute_input":"2024-04-04T05:14:02.334300Z","iopub.status.idle":"2024-04-04T05:14:17.225147Z","shell.execute_reply.started":"2024-04-04T05:14:02.334271Z","shell.execute_reply":"2024-04-04T05:14:17.224073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I will convert the polars data frame to pandas so that pandas methods work later.  It seems mroe memory efficient to load as polars, and then convert to pandas and load as pandas.","metadata":{}},{"cell_type":"code","source":"train = train.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:14:38.905590Z","iopub.execute_input":"2024-04-04T05:14:38.905982Z","iopub.status.idle":"2024-04-04T05:14:38.936066Z","shell.execute_reply.started":"2024-04-04T05:14:38.905955Z","shell.execute_reply":"2024-04-04T05:14:38.933959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#get list of ids fro submission file\nids = test['case_id'].tolist()\nprint(ids)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:14:41.446839Z","iopub.execute_input":"2024-04-04T05:14:41.447269Z","iopub.status.idle":"2024-04-04T05:14:41.453822Z","shell.execute_reply.started":"2024-04-04T05:14:41.447236Z","shell.execute_reply":"2024-04-04T05:14:41.452748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# only select common columns to use\ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest = test[common_columns]\n\n#subset train with only columns seen in test + target\ntrain = train[common_columns+['target']]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:14:44.233062Z","iopub.execute_input":"2024-04-04T05:14:44.233985Z","iopub.status.idle":"2024-04-04T05:14:45.594654Z","shell.execute_reply.started":"2024-04-04T05:14:44.233944Z","shell.execute_reply":"2024-04-04T05:14:45.593314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n#fit on stratified sample, 0.005%\n#note no random seed, warning is fine\n\ntrain_sample = train.groupby('target', group_keys=False).apply(lambda x: x.sample(frac=0.5)).reset_index(drop=True)\ntrain_sample.head()\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a data frame y that contains only the 'target' column from the train or train_sample data frame\n#create a data frame X that drops the 'target' column from the train or train_sample data frame\n \ny = train.loc[:, 'target'].to_frame('target')\nX = train.drop(['target',], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:16:54.742306Z","iopub.execute_input":"2024-04-04T05:16:54.742796Z","iopub.status.idle":"2024-04-04T05:16:55.832899Z","shell.execute_reply.started":"2024-04-04T05:16:54.742761Z","shell.execute_reply":"2024-04-04T05:16:55.831650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check target distribution\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(),4))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:17:01.922334Z","iopub.execute_input":"2024-04-04T05:17:01.922740Z","iopub.status.idle":"2024-04-04T05:17:01.959985Z","shell.execute_reply.started":"2024-04-04T05:17:01.922711Z","shell.execute_reply":"2024-04-04T05:17:01.958850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del train\n#gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#remove case_id and week_num from numeric\nnumeric_cols = test.select_dtypes(include=['number']).columns.tolist()\nnumeric_cols.remove('case_id')\nnumeric_cols.remove('WEEK_NUM')","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:17:09.425059Z","iopub.execute_input":"2024-04-04T05:17:09.425858Z","iopub.status.idle":"2024-04-04T05:17:09.434827Z","shell.execute_reply.started":"2024-04-04T05:17:09.425821Z","shell.execute_reply":"2024-04-04T05:17:09.433712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(numeric_cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:17:12.111566Z","iopub.execute_input":"2024-04-04T05:17:12.111957Z","iopub.status.idle":"2024-04-04T05:17:12.117340Z","shell.execute_reply.started":"2024-04-04T05:17:12.111929Z","shell.execute_reply":"2024-04-04T05:17:12.116260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nX.head()\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:17:18.572997Z","iopub.execute_input":"2024-04-04T05:17:18.573403Z","iopub.status.idle":"2024-04-04T05:17:26.544177Z","shell.execute_reply.started":"2024-04-04T05:17:18.573374Z","shell.execute_reply":"2024-04-04T05:17:26.543256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:17:28.763407Z","iopub.execute_input":"2024-04-04T05:17:28.763806Z","iopub.status.idle":"2024-04-04T05:17:28.797645Z","shell.execute_reply.started":"2024-04-04T05:17:28.763778Z","shell.execute_reply":"2024-04-04T05:17:28.796458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:17:31.235109Z","iopub.execute_input":"2024-04-04T05:17:31.235556Z","iopub.status.idle":"2024-04-04T05:17:31.245213Z","shell.execute_reply.started":"2024-04-04T05:17:31.235522Z","shell.execute_reply":"2024-04-04T05:17:31.243958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n#75/25 split\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.25, stratify=y, random_state=123)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del X, y\n#gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n#check proper split\nprint(X_train.shape)\nprint(X_valid.shape)\nprint(y_train.shape)\nprint(y_valid.shape)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(y_valid['target'].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search = RandomForestClassifier(criterion='entropy', n_estimators = 500, class_weight = 'balanced', random_state=100)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:18:22.336127Z","iopub.execute_input":"2024-04-04T05:18:22.336538Z","iopub.status.idle":"2024-04-04T05:18:22.342557Z","shell.execute_reply.started":"2024-04-04T05:18:22.336509Z","shell.execute_reply":"2024-04-04T05:18:22.341504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here I will drop case_id and week_num from my features, for purposes of training only.  I will leave the original X_train and X_valid for metric scoring later.","metadata":{}},{"cell_type":"code","source":"\"\"\"\nweeks = X[\"WEEK_NUM\"]\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\ntest.drop(['case_id', 'WEEK_NUM'], axis=1, inplace = True)\n# X_valid_feats = X_valid.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n#sort columns in alphabetical order for training so that columns match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\ntest=test.reindex(sorted(test.columns), axis=1)\nprint(X_feats.shape)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop case_id and week_num from features, leave original X_train and X_valid for metric scoring later\nweeks = X[\"WEEK_NUM\"]\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\n# X_valid_feats = X_valid.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n# Sort columns in alphabetical order for training so columns match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\n# X_valid_feats = X_valid_feats.reindex(sorted(X_valid_feats.columns), axis=1)\n\nprint(X_feats.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:18:56.673687Z","iopub.execute_input":"2024-04-04T05:18:56.674156Z","iopub.status.idle":"2024-04-04T05:19:00.909551Z","shell.execute_reply.started":"2024-04-04T05:18:56.674120Z","shell.execute_reply":"2024-04-04T05:19:00.908389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(['case_id', 'WEEK_NUM'], axis=1, inplace=True)\ntest=test.reindex(sorted(test.columns), axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:23:32.404428Z","iopub.execute_input":"2024-04-04T05:23:32.405028Z","iopub.status.idle":"2024-04-04T05:23:34.196058Z","shell.execute_reply.started":"2024-04-04T05:23:32.404998Z","shell.execute_reply":"2024-04-04T05:23:34.194999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Drop case_id and week_num from features, leave original X_train and X_valid for metric scoring later\nX_train_feats = X_train.drop(['case_id', 'WEEK_NUM'], axis=1)\nX_valid_feats = X_valid.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n# Sort columns in alphabetical order for training so columns match test submission\nX_train_feats = X_train_feats.reindex(sorted(X_train_feats.columns), axis=1)\nX_valid_feats = X_valid_feats.reindex(sorted(X_valid_feats.columns), axis=1)\n\nprint(X_train_feats.shape)\nprint(X_valid_feats.shape)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train_feats.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(list(X_train_feats))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_feats.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:19:37.385422Z","iopub.execute_input":"2024-04-04T05:19:37.386163Z","iopub.status.idle":"2024-04-04T05:19:37.419184Z","shell.execute_reply.started":"2024-04-04T05:19:37.386129Z","shell.execute_reply":"2024-04-04T05:19:37.418227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PCA Analysis","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn.decomposition import PCA\n#initialize pca with 60 components\n\npca = PCA(n_components=60)\n\n#fit PCA on training data\npca.fit(X_feats)\n\n#transform both the training and validation sets\nX_feats_pca = pca.transform(X_feats)\ntest_pca = pca.transform(test)\n\n\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-04-04T05:20:57.234626Z","iopub.execute_input":"2024-04-04T05:20:57.235944Z","iopub.status.idle":"2024-04-04T05:21:51.843487Z","shell.execute_reply.started":"2024-04-04T05:20:57.235891Z","shell.execute_reply":"2024-04-04T05:21:51.842447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.drop(['case_id', 'WEEK_NUM'], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.reindex(sorted(test.columns), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pca=pca.transform(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##del X, X_feats, test\n##gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_feats_pca)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a scree plot\nexplained_variance = pca.explained_variance_ratio_\ncomponents = np.arange(1, len(explained_variance) + 1)\n\nplt.figure(figsize=(24, 12))\nplt.plot(components, explained_variance, marker='o', linestyle='-')\nplt.title('Scree Plot')\nplt.xlabel('Principal Component')\nplt.ylabel('Explained Variance Ratio')\nplt.xticks(components)\nplt.grid(True)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculate the cumulative explained variance\ncumulative_variance_ratio = np.cumsum(pca.explained_variance_ratio_)\nprint(cumulative_variance_ratio)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cumulative_variance_40_comp = cumulative_variance_ratio[40]\nprint(cumulative_variance_40_comp)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fit Random Forest Model with PCA Features","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\ngrid_search.fit(X_feats_pca, y_train)\n\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\ny_pred = grid_search.predict(X_valid_pca)\n\nprint(f'Validation Target: {round(y_valid.target.value_counts()[1]/y_valid.target.value_counts().sum(),4)}')\nprint(f'Validation Accuracy: {accuracy_score(y_valid,y_pred)}')\nwarnings.filterwarnings('default')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n# Print confusion matrix with percent\ntarget_names= ['No Default', 'Default']\nmatrix = confusion_matrix(y_valid, y_pred, normalize='true')\ncm_display = ConfusionMatrixDisplay(confusion_matrix= matrix, display_labels=target_names).plot()\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n# Print out the report\nprint(classification_report(y_valid, y_pred, target_names = target_names))\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot ROC AUC curve using skplt\nwarnings.filterwarnings(\"ignore\")\ny_prob = grid_search.predict_proba(X_valid_pca)\nskplt.metrics.plot_roc_curve(y_valid, y_prob)\nplt.show()\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Metric Scoring","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n\nbase_train = pd.concat([X_train, y_train], axis=1)\nbase_train['score'] = grid_search.predict_proba(X_feats_pca)[:,1]\nprint(f\"The AUC score on the train set is: {roc_auc_score(base_train['target'], base_train['score'])}\")\n\nbase_valid = pd.concat([X_valid, y_valid], axis=1)\nbase_valid['score'] = grid_search.predict_proba(X_valid_pca)[:,1]\nprint(f\"The AUC score on the valid set is: {roc_auc_score(base_valid['target'], base_valid['score'])}\")\n\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken from competition starter notebook made by one of the organizers \n# Note: may not work based on how we sampled a low percent of the data because some weeks will be all 0 targets\n# Using 1 and 2% samples did not work\n# Using 5% only worked for train, not for valid\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del X_train_feats, X_valid_feats, X_train, X_valid, y_train, y_valid\n#gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"# Sort columns alphabetically to match loaded model\ntest = test.reindex(sorted(test.columns), axis=1)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions=grid_search.predict_proba(test_pca)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.DataFrame({'case_id': ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fit LGBM Model with pca features","metadata":{}},{"cell_type":"code","source":"\"\"\"\n%%time\nwarnings.filterwarnings(\"ignore\")\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nfitted_models = []\ncv_scores = []\n\n#default boosting type: gbdt\n\ngrid_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"class_weight\": \"balanced\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,\n    \"colsample_bytree\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 123, \n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\": True,\n    'num_leaves': 64\n}\n\nfor idx_train, idx_valid in cv.split(X_train_pca, y, groups=weeks):\n    X_train, y_train = X_feats.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X_feats.iloc[idx_valid], y.iloc[idx_valid]\n    \n    clf = lgb.LGBMClassifier(**grid_params)\n    clf.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(100)])\n    fitted_models.append(clf)\n    \n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores.append(auc_score)\n    \nprint(\"CV AUC scores: \", cv_scores)\nprint(\"Maximum CV AUC score: \", max(cv_scores))\n\nwarnings.filterwarnings(\"default\")\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## model = fitted_models[np.argmax(cv_scores)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PCA Analysis","metadata":{}},{"cell_type":"markdown","source":"### Logistic Regression","metadata":{}},{"cell_type":"code","source":"#from sklearn.linear_model import LogisticRegression\n#from sklearn.metrics import classification_report\n#from sklearn.utils.class_weight import compute_class_weight","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model=LogisticRegression(class_weight='balanced')\n#model.fit(X_train_pca, y_train)\n\n#y_pred=model.predict(X_valid_pca)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#accuracy=accuracy_score(y_valid, y_pred)\n#print(accuracy)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#report=classification_report(y_valid, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(report)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##from imblearn.over_sampling import SMOTE\n##sm = SMOTE(random_state=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##smote = SMOTE()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##X_resampled, y_resampled = smote.fit_resample(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##print(y_resampled['target'].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#now split the resampled dataset into train/test sets\n\n# X_train, X_valid, y_train, y_valid = train_test_split(X_resampled, y_resampled, test_size=0.2, random_state=5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check proper split\n#print(X_train.shape)\n#print(X_valid.shape)\n#print(y_train.shape)\n#print(y_valid.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MODEL BUILDING","metadata":{}},{"cell_type":"markdown","source":"### Logistic Regression","metadata":{}},{"cell_type":"code","source":"##lr = LogisticRegression()\n##lr_params = {\n    ##'solver': ('lbfgs','newton-cg','newton-cholesky','sag'), # these 4 solvers either use l2 or None penalty\n   ## 'penalty': ('l2', None), 'C': (0.01, 0.1, 1, 10, 100), # Regularization parameter, default = 1.0\n##}\n\n#Using roc_auc instead of default accuracy to score due to imbalanced target\n#Competition evaluation metric is based off AUC\n##lr_search = GridSearchCV(lr, lr_params, verbose=1, scoring='roc_auc', cv=3)\n##print(\"Hyperparameters to tune are:\")\n##pprint(lr_params)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##%%time\n##warnings.filterwarnings(\"ignore\")\n##lr_search.fit(X_train_pca, y_train_pca)\n##warnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(f\"Best score= {lr_search.best_score_:0.3f}\")\n#print(\"Best parameters set:\")\n#best_parameters = lr_search.best_estimator_.get_params()\n#for name in sorted(lr_params.keys()):\n#    print(\"\\t%s: %r\" % (name, best_parameters[name]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#warnings.filterwarnings(\"ignore\")\n#y_pred = lr_search.predict(X_valid)\n\n#print(f'Validation Target: {round(y_valid.target.value_counts()[1]/y_valid.target.value_counts().sum(),4)}')\n#print(f'Validation Accuracy: {accuracy_score(y_valid,y_pred)}')\n#warnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#warnings.filterwarnings(\"ignore\")\n# Print confusion matrix with percent\n#target_names= ['No Default', 'Default']\n#matrix = confusion_matrix(y_valid, y_pred, normalize='true')\n#cm_display = ConfusionMatrixDisplay(confusion_matrix= matrix, display_labels=target_names).plot()\n#warnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#warnings.filterwarnings(\"ignore\")\n# Print out the report\n#print(classification_report(y_valid, y_pred, target_names = target_names))\n#warnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save the best model parameters\n#lr_model1 = lr_search.best_estimator_\n#print(lr_model1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save model\n#joblib.dump(lr_model1, 'lr_model1.joblib')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ntest = test.reindex(sorted(test.columns), axis=1)\n\npredictions = model.predict_proba(test.drop(['case_id', 'WEEK_NUM'], axis=1))\nprint(predictions)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id':ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Random Forest Model","metadata":{}},{"cell_type":"code","source":"#random_forest_classifier = RandomForestClassifier(n_estimators=10, random_state=10)\n#random_forest_classifier.fit(X_train, y_train.values.ravel())","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}