{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7870556,"sourceType":"datasetVersion","datasetId":4618198},{"sourceId":7941325,"sourceType":"datasetVersion","datasetId":4506020}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nfrom pprint import pprint\nimport scikitplot as skplt\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, confusion_matrix, ConfusionMatrixDisplay, classification_report, roc_auc_score\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, StratifiedGroupKFold\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import SVC\n\nfrom contextlib import suppress\n\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.decomposition import PCA\n\nfrom pprint import pprint\nimport lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:54:59.111006Z","iopub.execute_input":"2024-03-30T15:54:59.111386Z","iopub.status.idle":"2024-03-30T15:55:01.001187Z","shell.execute_reply.started":"2024-03-30T15:54:59.111356Z","shell.execute_reply":"2024-03-30T15:55:00.999970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now I will define my User Defined Functions","metadata":{}},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\n#this function takes a data frame as input, and iterates over all the columns\n#each column ending with a P or an A, it casts the column to Float64\ndef set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated columns as Float64, tried combining sum and max, but did not work correctly\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\n#this function takes a data frame as input, and iterates over all the columns\n#each columns that is a string or an object...it converts the col to string type, then to cat type\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n#This function prints the percent of missing values\n# Changed this function to work for Pandas\ndef missing_values(df, threshold = 0.0):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == 'float64':\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.003075Z","iopub.execute_input":"2024-03-30T15:55:01.003616Z","iopub.status.idle":"2024-03-30T15:55:01.122773Z","shell.execute_reply.started":"2024-03-30T15:55:01.003583Z","shell.execute_reply":"2024-03-30T15:55:01.121734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Join the test data","metadata":{}},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway +  \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.124332Z","iopub.execute_input":"2024-03-30T15:55:01.124671Z","iopub.status.idle":"2024-03-30T15:55:01.173566Z","shell.execute_reply.started":"2024-03-30T15:55:01.124645Z","shell.execute_reply":"2024-03-30T15:55:01.172396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional depth=1 files\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\n# test_debitcard_1 = pl.read_csv(pathway + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.176205Z","iopub.execute_input":"2024-03-30T15:55:01.176808Z","iopub.status.idle":"2024-03-30T15:55:01.187444Z","shell.execute_reply.started":"2024-03-30T15:55:01.176768Z","shell.execute_reply":"2024-03-30T15:55:01.186634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use aggregation functions in tables with depth >=1\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.190669Z","iopub.execute_input":"2024-03-30T15:55:01.191546Z","iopub.status.idle":"2024-03-30T15:55:01.201840Z","shell.execute_reply.started":"2024-03-30T15:55:01.191517Z","shell.execute_reply":"2024-03-30T15:55:01.201077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"),\n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.203126Z","iopub.execute_input":"2024-03-30T15:55:01.203765Z","iopub.status.idle":"2024-03-30T15:55:01.216667Z","shell.execute_reply.started":"2024-03-30T15:55:01.203736Z","shell.execute_reply":"2024-03-30T15:55:01.215568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.218074Z","iopub.execute_input":"2024-03-30T15:55:01.218527Z","iopub.status.idle":"2024-03-30T15:55:01.235731Z","shell.execute_reply.started":"2024-03-30T15:55:01.218487Z","shell.execute_reply":"2024-03-30T15:55:01.234346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n\n# drop_cols = list_missing(test)\ndrop_cols = ['avgdbddpdlast3m_4187120P', 'avgdbdtollast24m_4525197P', 'avglnamtstart24m_4525187A', 'avgpmtlast12m_4525200A', 'bankacctype_710L', 'cardtype_51L', 'clientscnt_136L', 'datelastinstal40dpd_247D', 'dtlastpmtallstes_4499206D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'inittransactionamount_650A', 'interestrategrace_34L', 'isbidproductrequest_292L', 'isdebitcard_729L', 'lastdelinqdate_224D', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'maxdbddpdlast1m_3658939P', 'maxlnamtstart6m_4525199A', 'maxpmtlast3m_4525190A', 'mindbdtollast24m_4525191P', 'payvacationpostpone_4187118D', 'totinstallast1m_4525188A', 'typesuite_864L', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'birthdate_574D', 'contractssum_5085716L', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'pmtscount_423L', 'pmtssum_45A', 'responsedate_4917613D', 'riskassesment_302T', 'riskassesment_940T', 'empl_employedfrom_271D', 'empl_industry_691L', 'empl_employedtotal_800L', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\n# print(drop_cols)\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.237504Z","iopub.execute_input":"2024-03-30T15:55:01.238274Z","iopub.status.idle":"2024-03-30T15:55:01.264582Z","shell.execute_reply.started":"2024-03-30T15:55:01.238233Z","shell.execute_reply":"2024-03-30T15:55:01.263822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1,test_credit_bureau_b_1,test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats   \ngc.collect()  ","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.265953Z","iopub.execute_input":"2024-03-30T15:55:01.266269Z","iopub.status.idle":"2024-03-30T15:55:01.380510Z","shell.execute_reply.started":"2024-03-30T15:55:01.266242Z","shell.execute_reply":"2024-03-30T15:55:01.379246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 2","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.384361Z","iopub.execute_input":"2024-03-30T15:55:01.384696Z","iopub.status.idle":"2024-03-30T15:55:01.417527Z","shell.execute_reply.started":"2024-03-30T15:55:01.384670Z","shell.execute_reply":"2024-03-30T15:55:01.416667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.418740Z","iopub.execute_input":"2024-03-30T15:55:01.419611Z","iopub.status.idle":"2024-03-30T15:55:01.428363Z","shell.execute_reply.started":"2024-03-30T15:55:01.419571Z","shell.execute_reply":"2024-03-30T15:55:01.427329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    pl.col(\"byoccupationinc_3656910L\").max().alias(\"byoccupationinc_3656910L_max\"),\n    pl.col(\"childnum_21L\").max().alias(\"childnum_21L_max\"),\n    pl.col(\"credacc_credlmt_575A\").max().alias(\"credacc_credlmt_575A_max\"),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"))\n\ntest_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))\n\ntest_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.430025Z","iopub.execute_input":"2024-03-30T15:55:01.430556Z","iopub.status.idle":"2024-03-30T15:55:01.449295Z","shell.execute_reply.started":"2024-03-30T15:55:01.430517Z","shell.execute_reply":"2024-03-30T15:55:01.448227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1,test_credit_bureau_a_1\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.450496Z","iopub.execute_input":"2024-03-30T15:55:01.450831Z","iopub.status.idle":"2024-03-30T15:55:01.568503Z","shell.execute_reply.started":"2024-03-30T15:55:01.450803Z","shell.execute_reply":"2024-03-30T15:55:01.567322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','byoccupationinc_3656910L_max','familystate_726L', 'amount_4527230A_sum', 'amount_4917619A_sum', 'pmtamount_36A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.570140Z","iopub.execute_input":"2024-03-30T15:55:01.570718Z","iopub.status.idle":"2024-03-30T15:55:01.593588Z","shell.execute_reply.started":"2024-03-30T15:55:01.570677Z","shell.execute_reply":"2024-03-30T15:55:01.592395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.595348Z","iopub.execute_input":"2024-03-30T15:55:01.596209Z","iopub.status.idle":"2024-03-30T15:55:01.716814Z","shell.execute_reply.started":"2024-03-30T15:55:01.596178Z","shell.execute_reply":"2024-03-30T15:55:01.715526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 3","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 2\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\n\nsel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.718277Z","iopub.execute_input":"2024-03-30T15:55:01.718683Z","iopub.status.idle":"2024-03-30T15:55:01.762908Z","shell.execute_reply.started":"2024-03-30T15:55:01.718650Z","shell.execute_reply":"2024-03-30T15:55:01.761699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.764474Z","iopub.execute_input":"2024-03-30T15:55:01.765528Z","iopub.status.idle":"2024-03-30T15:55:01.778248Z","shell.execute_reply.started":"2024-03-30T15:55:01.765481Z","shell.execute_reply":"2024-03-30T15:55:01.776913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2, test_person_2, test_credit_bureau_a_2\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.779931Z","iopub.execute_input":"2024-03-30T15:55:01.780432Z","iopub.status.idle":"2024-03-30T15:55:01.903489Z","shell.execute_reply.started":"2024-03-30T15:55:01.780374Z","shell.execute_reply":"2024-03-30T15:55:01.902606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\").join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.904800Z","iopub.execute_input":"2024-03-30T15:55:01.905124Z","iopub.status.idle":"2024-03-30T15:55:01.920833Z","shell.execute_reply.started":"2024-03-30T15:55:01.905097Z","shell.execute_reply":"2024-03-30T15:55:01.919606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:01.922051Z","iopub.execute_input":"2024-03-30T15:55:01.922383Z","iopub.status.idle":"2024-03-30T15:55:02.039496Z","shell.execute_reply.started":"2024-03-30T15:55:01.922355Z","shell.execute_reply":"2024-03-30T15:55:02.038046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####join_data3.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.040799Z","iopub.execute_input":"2024-03-30T15:55:02.041193Z","iopub.status.idle":"2024-03-30T15:55:02.047853Z","shell.execute_reply.started":"2024-03-30T15:55:02.041152Z","shell.execute_reply":"2024-03-30T15:55:02.046822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n\n# Convert back to polars for date extraction \njoin_test = pl.from_pandas(join_test)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.049384Z","iopub.execute_input":"2024-03-30T15:55:02.049898Z","iopub.status.idle":"2024-03-30T15:55:02.088463Z","shell.execute_reply.started":"2024-03-30T15:55:02.049858Z","shell.execute_reply":"2024-03-30T15:55:02.087118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Join of the Test Data","metadata":{}},{"cell_type":"code","source":"join_test = join_test.with_columns(pl.col('date_decision','birth_259D').cast(pl.Date))\n#Feature engineer days diff between decision and birthdate\n\ntest = join_test.with_columns(\n    ((pl.col(\"date_decision\") - pl.col(\"birth_259D\")) / (24*60*60*1000)).cast(pl.Float64).alias(\"date_diff\"))\n\n\n\n\n# Drop uneeded date columns plus other columns\ndate_list = ['date_decision','MONTH','firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D', 'firstclxcampaign_1125D', \n'birth_259D','datefirstoffer_1144D', 'datelastunpaid_3546854D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'responsedate_1012D', 'responsedate_4527233D', 'requesttype_4525192L']\n\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(date_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test\ngc.collect()\n\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.092765Z","iopub.execute_input":"2024-03-30T15:55:02.093122Z","iopub.status.idle":"2024-03-30T15:55:02.241618Z","shell.execute_reply.started":"2024-03-30T15:55:02.093094Z","shell.execute_reply":"2024-03-30T15:55:02.240835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noticed some of these numeric variables were parsed as strings, changed them all to int \nnum_list = ['days120_123L','days180_256L','days30_165L','days360_512L','days90_310L','firstquarter_103L','numinstpaidlastcontr_4325080L',\n            'fourthquarter_440L','numinstlswithdpd5_4187116L','numberofqueries_373L', 'secondquarter_766L','thirdquarter_1082L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')\n\n# Drop some unneeded columns, based on training missing values, 0 range for numeric, or one unique category\ndrop_list = ['lastapprcommoditytypec_5251766M', 'lastrejectcommodtypec_5251769M','lastrejectcommoditycat_161M','lastrejectreasonclient_4145040M',\n            'previouscontdistrict_112M','lastapprcommoditycat_1041M', 'lastcancelreason_561M','lastrejectreason_759M', 'addres_zip_823M', 'addres_district_368M',\n            'commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L','paytype1st_925L_OTHER','paytype_783L_OTHER','applicationcnt_361L','empls_employer_name_740M_a55475b1']\n\n\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.242793Z","iopub.execute_input":"2024-03-30T15:55:02.243648Z","iopub.status.idle":"2024-03-30T15:55:02.282299Z","shell.execute_reply.started":"2024-03-30T15:55:02.243617Z","shell.execute_reply":"2024-03-30T15:55:02.281207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imputation","metadata":{}},{"cell_type":"code","source":"print(np.count_nonzero(test.isnull()))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.283858Z","iopub.execute_input":"2024-03-30T15:55:02.284281Z","iopub.status.idle":"2024-03-30T15:55:02.292783Z","shell.execute_reply.started":"2024-03-30T15:55:02.284241Z","shell.execute_reply":"2024-03-30T15:55:02.291485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = imputer(test)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.294381Z","iopub.execute_input":"2024-03-30T15:55:02.294845Z","iopub.status.idle":"2024-03-30T15:55:02.395322Z","shell.execute_reply.started":"2024-03-30T15:55:02.294808Z","shell.execute_reply":"2024-03-30T15:55:02.394206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.count_nonzero(test.isnull()))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.397180Z","iopub.execute_input":"2024-03-30T15:55:02.397632Z","iopub.status.idle":"2024-03-30T15:55:02.408827Z","shell.execute_reply.started":"2024-03-30T15:55:02.397592Z","shell.execute_reply":"2024-03-30T15:55:02.407459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I can now see that I reduced the missing value count from 457 to 0 after the imputation.","metadata":{}},{"cell_type":"code","source":"# See boolean columns\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.414623Z","iopub.execute_input":"2024-03-30T15:55:02.414956Z","iopub.status.idle":"2024-03-30T15:55:02.424507Z","shell.execute_reply.started":"2024-03-30T15:55:02.414929Z","shell.execute_reply":"2024-03-30T15:55:02.423334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include=['category']).columns.tolist()\n#create dummies for all cat columns, not dropping first to keep column names the same as training\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.426355Z","iopub.execute_input":"2024-03-30T15:55:02.426835Z","iopub.status.idle":"2024-03-30T15:55:02.496869Z","shell.execute_reply.started":"2024-03-30T15:55:02.426795Z","shell.execute_reply":"2024-03-30T15:55:02.495802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Model","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv('/kaggle/input/training/train_final_dummy.csv')\ntrain.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:02.498691Z","iopub.execute_input":"2024-03-30T15:55:02.499139Z","iopub.status.idle":"2024-03-30T15:55:11.944646Z","shell.execute_reply.started":"2024-03-30T15:55:02.499101Z","shell.execute_reply":"2024-03-30T15:55:11.943490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I will convert the polars data frame to pandas so that pandas methods work later.  It seems mroe memory efficient to load as polars, and then convert to pandas and load as pandas.","metadata":{}},{"cell_type":"code","source":"train = train.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:11.946088Z","iopub.execute_input":"2024-03-30T15:55:11.947286Z","iopub.status.idle":"2024-03-30T15:55:15.097934Z","shell.execute_reply.started":"2024-03-30T15:55:11.947243Z","shell.execute_reply":"2024-03-30T15:55:15.096742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get list of ids fro submission file\nids = test['case_id'].tolist()\nprint(ids)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:15.100965Z","iopub.execute_input":"2024-03-30T15:55:15.102150Z","iopub.status.idle":"2024-03-30T15:55:15.107777Z","shell.execute_reply.started":"2024-03-30T15:55:15.102111Z","shell.execute_reply":"2024-03-30T15:55:15.106688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# only select common columns to use\ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest = test[common_columns]\n\n#subset train with only columns seen in test + target\ntrain = train[common_columns+['target']]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:15.109185Z","iopub.execute_input":"2024-03-30T15:55:15.109548Z","iopub.status.idle":"2024-03-30T15:55:15.977082Z","shell.execute_reply.started":"2024-03-30T15:55:15.109518Z","shell.execute_reply":"2024-03-30T15:55:15.975814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n#fit on stratified sample, 0.5%\n#note no random seed, warning is fine\n\ntrain_sample = train.groupby('target', group_keys=False).apply(lambda x: x.sample(frac=0.01)).reset_index(drop=True)\ntrain_sample.head()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:15.978751Z","iopub.execute_input":"2024-03-30T15:55:15.979103Z","iopub.status.idle":"2024-03-30T15:55:17.675529Z","shell.execute_reply.started":"2024-03-30T15:55:15.979075Z","shell.execute_reply":"2024-03-30T15:55:17.674400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a data frame y that contains only the 'target' column from the train_sample data frame\n#create a data frame X that drops the 'target' column from the train_sample data frame\n \ny = train.loc[:, 'target'].to_frame('target')\nX = train.drop(['target',], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:17.677224Z","iopub.execute_input":"2024-03-30T15:55:17.677583Z","iopub.status.idle":"2024-03-30T15:55:17.688766Z","shell.execute_reply.started":"2024-03-30T15:55:17.677554Z","shell.execute_reply":"2024-03-30T15:55:17.687613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check target distribution\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(),4))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:17.690469Z","iopub.execute_input":"2024-03-30T15:55:17.690795Z","iopub.status.idle":"2024-03-30T15:55:17.698897Z","shell.execute_reply.started":"2024-03-30T15:55:17.690769Z","shell.execute_reply":"2024-03-30T15:55:17.697781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:17.700243Z","iopub.execute_input":"2024-03-30T15:55:17.700595Z","iopub.status.idle":"2024-03-30T15:55:17.832873Z","shell.execute_reply.started":"2024-03-30T15:55:17.700564Z","shell.execute_reply":"2024-03-30T15:55:17.831641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#remove case_id and week_num from numeric\nnumeric_cols = test.select_dtypes(include=['number']).columns.tolist()\nnumeric_cols.remove('case_id')\nnumeric_cols.remove('WEEK_NUM')","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:17.834500Z","iopub.execute_input":"2024-03-30T15:55:17.834832Z","iopub.status.idle":"2024-03-30T15:55:17.843534Z","shell.execute_reply.started":"2024-03-30T15:55:17.834806Z","shell.execute_reply":"2024-03-30T15:55:17.842365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nX.head()\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:17.845089Z","iopub.execute_input":"2024-03-30T15:55:17.845401Z","iopub.status.idle":"2024-03-30T15:55:17.958399Z","shell.execute_reply.started":"2024-03-30T15:55:17.845375Z","shell.execute_reply":"2024-03-30T15:55:17.957347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here I will drop case_id and week_num from my features, for purposes of training only.  I will leave the original X_train and X_valid for metric scoring later.","metadata":{}},{"cell_type":"code","source":"weeks = X[\"WEEK_NUM\"]\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\ntest.drop(['case_id', 'WEEK_NUM'], axis=1, inplace=True)\n\n#sort columns in alphabetical order for training so that columns match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\ntest = test.reindex(sorted(test.columns), axis=1)\n\nprint(X_feats.shape)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:17.959881Z","iopub.execute_input":"2024-03-30T15:55:17.960224Z","iopub.status.idle":"2024-03-30T15:55:18.012798Z","shell.execute_reply.started":"2024-03-30T15:55:17.960195Z","shell.execute_reply":"2024-03-30T15:55:18.011484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test.shape)\nX_feats.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:18.013997Z","iopub.execute_input":"2024-03-30T15:55:18.014317Z","iopub.status.idle":"2024-03-30T15:55:18.049050Z","shell.execute_reply.started":"2024-03-30T15:55:18.014289Z","shell.execute_reply":"2024-03-30T15:55:18.047903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PCA Analysis","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n#initialize pca with 60 components\n\npca = PCA(n_components=60)\n\n#fit PCA on training data\npca.fit(X_feats)\n\n#transform both the training and validation sets\nX_feats_pca = pca.transform(X_feats)\ntest_pca = pca.transform(test)\n\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:57:31.607204Z","iopub.execute_input":"2024-03-30T15:57:31.608106Z","iopub.status.idle":"2024-03-30T15:57:31.882947Z","shell.execute_reply.started":"2024-03-30T15:57:31.608058Z","shell.execute_reply":"2024-03-30T15:57:31.881683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X, X_feats, test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:18.957133Z","iopub.execute_input":"2024-03-30T15:55:18.961165Z","iopub.status.idle":"2024-03-30T15:55:19.116276Z","shell.execute_reply.started":"2024-03-30T15:55:18.961110Z","shell.execute_reply":"2024-03-30T15:55:19.114902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_feats_pca)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:19.117708Z","iopub.execute_input":"2024-03-30T15:55:19.118145Z","iopub.status.idle":"2024-03-30T15:55:19.126354Z","shell.execute_reply.started":"2024-03-30T15:55:19.118106Z","shell.execute_reply":"2024-03-30T15:55:19.125157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a scree plot\nexplained_variance = pca.explained_variance_ratio_\ncomponents = np.arange(1, len(explained_variance) + 1)\n\nplt.figure(figsize=(24, 12))\nplt.plot(components, explained_variance, marker='o', linestyle='-')\nplt.title('Scree Plot')\nplt.xlabel('Principal Component')\nplt.ylabel('Explained Variance Ratio')\nplt.xticks(components)\nplt.grid(True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:19.127904Z","iopub.execute_input":"2024-03-30T15:55:19.128344Z","iopub.status.idle":"2024-03-30T15:55:20.107003Z","shell.execute_reply.started":"2024-03-30T15:55:19.128306Z","shell.execute_reply":"2024-03-30T15:55:20.104088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculate the cumulative explained variance\ncumulative_variance_ratio = np.cumsum(pca.explained_variance_ratio_)\nprint(cumulative_variance_ratio)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:20.112119Z","iopub.execute_input":"2024-03-30T15:55:20.114284Z","iopub.status.idle":"2024-03-30T15:55:20.134424Z","shell.execute_reply.started":"2024-03-30T15:55:20.114200Z","shell.execute_reply":"2024-03-30T15:55:20.129718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cumulative_variance_ratio[40])","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:20.140766Z","iopub.execute_input":"2024-03-30T15:55:20.142983Z","iopub.status.idle":"2024-03-30T15:55:20.162180Z","shell.execute_reply.started":"2024-03-30T15:55:20.142645Z","shell.execute_reply":"2024-03-30T15:55:20.158924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fit LGBM Model with pca features","metadata":{}},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nfitted_models = []\ncv_scores = []\n\n#default boosting type: gbdt\n\ngrid_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,\n    \"colsample_bytree\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 123, \n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\": True,\n    'num_leaves': 64\n}\n\nfor idx_train, idx_valid in cv.split(X_feats_pca, y, groups=weeks):\n    X_train, y_train = X_feats_pca[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X_feats_pca[idx_valid], y.iloc[idx_valid]\n    \n    clf = lgb.LGBMClassifier(**grid_params)\n    clf.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(100)])\n    fitted_models.append(clf)\n    \n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores.append(auc_score)\n    \nprint(\"CV AUC scores: \", cv_scores)\nprint(\"Maximum CV AUC score: \", max(cv_scores))\n\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:20.166730Z","iopub.execute_input":"2024-03-30T15:55:20.169232Z","iopub.status.idle":"2024-03-30T15:55:43.667302Z","shell.execute_reply.started":"2024-03-30T15:55:20.169101Z","shell.execute_reply":"2024-03-30T15:55:43.666034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = fitted_models[np.argmax(cv_scores)]","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:43.668930Z","iopub.execute_input":"2024-03-30T15:55:43.669394Z","iopub.status.idle":"2024-03-30T15:55:43.679405Z","shell.execute_reply.started":"2024-03-30T15:55:43.669355Z","shell.execute_reply":"2024-03-30T15:55:43.678118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"predictions = model.predict_proba(test_pca)\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:43.680965Z","iopub.execute_input":"2024-03-30T15:55:43.681288Z","iopub.status.idle":"2024-03-30T15:55:43.689973Z","shell.execute_reply.started":"2024-03-30T15:55:43.681261Z","shell.execute_reply":"2024-03-30T15:55:43.688874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id':ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:43.691286Z","iopub.execute_input":"2024-03-30T15:55:43.691624Z","iopub.status.idle":"2024-03-30T15:55:43.708727Z","shell.execute_reply.started":"2024-03-30T15:55:43.691597Z","shell.execute_reply":"2024-03-30T15:55:43.707576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-30T15:55:43.709744Z","iopub.execute_input":"2024-03-30T15:55:43.710064Z","iopub.status.idle":"2024-03-30T15:55:43.724471Z","shell.execute_reply.started":"2024-03-30T15:55:43.710039Z","shell.execute_reply":"2024-03-30T15:55:43.723503Z"},"trusted":true},"execution_count":null,"outputs":[]}]}