{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-23T10:47:52.219988Z","iopub.execute_input":"2024-03-23T10:47:52.220505Z","iopub.status.idle":"2024-03-23T10:47:52.239459Z","shell.execute_reply.started":"2024-03-23T10:47:52.22046Z","shell.execute_reply":"2024-03-23T10:47:52.237941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nfrom pprint import pprint\nimport scikitplot as skplt\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, confusion_matrix, ConfusionMatrixDisplay, classification_report\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\nfrom imblearn.under_sampling import NearMiss\nfrom imblearn.over_sampling import SMOTE","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:47:58.944254Z","iopub.execute_input":"2024-03-23T10:47:58.945889Z","iopub.status.idle":"2024-03-23T10:47:58.956908Z","shell.execute_reply.started":"2024-03-23T10:47:58.945822Z","shell.execute_reply":"2024-03-23T10:47:58.955478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\ndef set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date\n        if col[-1] in (\"D\"):\n            df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated columns as Float64, tried combining sum and max, but did not work correctly\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n# Changed this function to work for Pandas\ndef missing_values(df, threshold = 0.0):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == 'float64':\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:48:02.359396Z","iopub.execute_input":"2024-03-23T10:48:02.360127Z","iopub.status.idle":"2024-03-23T10:48:02.375449Z","shell.execute_reply.started":"2024-03-23T10:48:02.360092Z","shell.execute_reply":"2024-03-23T10:48:02.373929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:52:31.128643Z","iopub.execute_input":"2024-03-23T04:52:31.129014Z","iopub.status.idle":"2024-03-23T04:52:31.144276Z","shell.execute_reply.started":"2024-03-23T04:52:31.128988Z","shell.execute_reply":"2024-03-23T04:52:31.143138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway +  \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:48:10.949564Z","iopub.execute_input":"2024-03-23T10:48:10.950006Z","iopub.status.idle":"2024-03-23T10:48:10.992392Z","shell.execute_reply.started":"2024-03-23T10:48:10.949976Z","shell.execute_reply":"2024-03-23T10:48:10.991121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional depth=1 files\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\n# test_debitcard_1 = pl.read_csv(pathway + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:48:14.919231Z","iopub.execute_input":"2024-03-23T10:48:14.919643Z","iopub.status.idle":"2024-03-23T10:48:14.934734Z","shell.execute_reply.started":"2024-03-23T10:48:14.919615Z","shell.execute_reply":"2024-03-23T10:48:14.93348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use aggregation functions in tables with depth >=1\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:48:19.008892Z","iopub.execute_input":"2024-03-23T10:48:19.009696Z","iopub.status.idle":"2024-03-23T10:48:19.020178Z","shell.execute_reply.started":"2024-03-23T10:48:19.009661Z","shell.execute_reply":"2024-03-23T10:48:19.018655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"),\n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:47:30.790544Z","iopub.execute_input":"2024-03-23T10:47:30.790978Z","iopub.status.idle":"2024-03-23T10:47:30.808417Z","shell.execute_reply.started":"2024-03-23T10:47:30.790948Z","shell.execute_reply":"2024-03-23T10:47:30.806588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:47:35.18417Z","iopub.execute_input":"2024-03-23T10:47:35.185539Z","iopub.status.idle":"2024-03-23T10:47:35.227367Z","shell.execute_reply.started":"2024-03-23T10:47:35.185492Z","shell.execute_reply":"2024-03-23T10:47:35.22595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n\ndrop_cols = ['avgdbddpdlast3m_4187120P', 'avgdbdtollast24m_4525197P', 'avglnamtstart24m_4525187A', 'avgpmtlast12m_4525200A', 'bankacctype_710L', 'cardtype_51L', 'clientscnt_136L', 'datelastinstal40dpd_247D', 'dtlastpmtallstes_4499206D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'inittransactionamount_650A', 'interestrategrace_34L', 'isbidproductrequest_292L', 'isdebitcard_729L', 'lastdelinqdate_224D', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'maxdbddpdlast1m_3658939P', 'maxlnamtstart6m_4525199A', 'maxpmtlast3m_4525190A', 'mindbdtollast24m_4525191P', 'payvacationpostpone_4187118D', 'totinstallast1m_4525188A', 'typesuite_864L', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'birthdate_574D', 'contractssum_5085716L', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'pmtscount_423L', 'pmtssum_45A', 'responsedate_4917613D', 'riskassesment_302T', 'riskassesment_940T', 'empl_employedfrom_271D', 'empl_industry_691L', 'empl_employedtotal_800L', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-23T10:47:45.091717Z","iopub.execute_input":"2024-03-23T10:47:45.092865Z","iopub.status.idle":"2024-03-23T10:47:45.390595Z","shell.execute_reply.started":"2024-03-23T10:47:45.092814Z","shell.execute_reply":"2024-03-23T10:47:45.388636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1,test_credit_bureau_b_1,test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats   \ngc.collect() ","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:52:55.368472Z","iopub.execute_input":"2024-03-23T04:52:55.369574Z","iopub.status.idle":"2024-03-23T04:52:55.466346Z","shell.execute_reply.started":"2024-03-23T04:52:55.369531Z","shell.execute_reply":"2024-03-23T04:52:55.465144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:02.799225Z","iopub.execute_input":"2024-03-23T04:53:02.799614Z","iopub.status.idle":"2024-03-23T04:53:02.832847Z","shell.execute_reply.started":"2024-03-23T04:53:02.799584Z","shell.execute_reply":"2024-03-23T04:53:02.831761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:08.338941Z","iopub.execute_input":"2024-03-23T04:53:08.339339Z","iopub.status.idle":"2024-03-23T04:53:08.347988Z","shell.execute_reply.started":"2024-03-23T04:53:08.339307Z","shell.execute_reply":"2024-03-23T04:53:08.347162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    pl.col(\"byoccupationinc_3656910L\").max().alias(\"byoccupationinc_3656910L_max\"),\n    pl.col(\"childnum_21L\").max().alias(\"childnum_21L_max\"),\n    pl.col(\"credacc_credlmt_575A\").max().alias(\"credacc_credlmt_575A_max\"),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"))\n\ntest_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))\n\ntest_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:10.140586Z","iopub.execute_input":"2024-03-23T04:53:10.140976Z","iopub.status.idle":"2024-03-23T04:53:10.168073Z","shell.execute_reply.started":"2024-03-23T04:53:10.140948Z","shell.execute_reply":"2024-03-23T04:53:10.167066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','byoccupationinc_3656910L_max','familystate_726L', 'amount_4527230A_sum', 'amount_4917619A_sum', 'pmtamount_36A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:16.438867Z","iopub.execute_input":"2024-03-23T04:53:16.439544Z","iopub.status.idle":"2024-03-23T04:53:16.457004Z","shell.execute_reply.started":"2024-03-23T04:53:16.43951Z","shell.execute_reply":"2024-03-23T04:53:16.456127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1,test_credit_bureau_a_1\ndel test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:19.11855Z","iopub.execute_input":"2024-03-23T04:53:19.118944Z","iopub.status.idle":"2024-03-23T04:53:19.211625Z","shell.execute_reply.started":"2024-03-23T04:53:19.118915Z","shell.execute_reply":"2024-03-23T04:53:19.210671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional testing data, depth = 2\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\n\nsel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:21.808994Z","iopub.execute_input":"2024-03-23T04:53:21.809375Z","iopub.status.idle":"2024-03-23T04:53:21.891153Z","shell.execute_reply.started":"2024-03-23T04:53:21.809348Z","shell.execute_reply":"2024-03-23T04:53:21.890152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:25.678702Z","iopub.execute_input":"2024-03-23T04:53:25.67907Z","iopub.status.idle":"2024-03-23T04:53:25.690655Z","shell.execute_reply.started":"2024-03-23T04:53:25.679032Z","shell.execute_reply":"2024-03-23T04:53:25.689349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:29.918488Z","iopub.execute_input":"2024-03-23T04:53:29.918891Z","iopub.status.idle":"2024-03-23T04:53:29.931821Z","shell.execute_reply.started":"2024-03-23T04:53:29.91886Z","shell.execute_reply":"2024-03-23T04:53:29.930838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2, test_person_2, test_credit_bureau_a_2\ndel test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:32.958753Z","iopub.execute_input":"2024-03-23T04:53:32.959347Z","iopub.status.idle":"2024-03-23T04:53:33.054773Z","shell.execute_reply.started":"2024-03-23T04:53:32.959316Z","shell.execute_reply":"2024-03-23T04:53:33.053821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n# Convert back to polars for datetime \njoin_test = pl.from_pandas(join_test)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:35.513778Z","iopub.execute_input":"2024-03-23T04:53:35.514676Z","iopub.status.idle":"2024-03-23T04:53:35.568834Z","shell.execute_reply.started":"2024-03-23T04:53:35.514635Z","shell.execute_reply":"2024-03-23T04:53:35.567554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_test = join_test.with_columns(pl.col('date_decision','birth_259D').cast(pl.Date))\n# Feature engineer days difference between decision and birthdate\ntest = join_test.with_columns(\n    ((pl.col(\"date_decision\") - pl.col(\"birth_259D\")) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(\"date_diff\"))\n\n# Drop uneeded date + other columns\ndate_list = ['date_decision','MONTH','firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D',\n             'birth_259D','datefirstoffer_1144D', 'datelastunpaid_3546854D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'responsedate_1012D', 'responsedate_4527233D', 'requesttype_4525192L']\n\"\"\"\n             ['avgoutstandbalancel6m_4187114A', 'firstclxcampaign_1125D', 'lastrejectcredamount_222A','maxdbddpdtollast6m_4187119P', 'maxdpdinstlnum_3546846P', 'maxoutstandbalancel12m_4187113A', 'numinstmatpaidtearly2d_4499204L', \n             'numinstpaid_4499208L', 'numinstpaidearly3dest_4493216L', 'numinstpaidearly5dest_4493211L', 'numinstpaidearly5dobd_4499205L', 'numinstpaidearlyest_4493214L', 'numinstregularpaidest_4493210L', \n             'numinsttopaygrest_4493213L', 'numinstunpaidmaxest_4493212L', 'sumoutstandtotalest_4493215A', 'familystate_447L']\n\"\"\"\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(date_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test\ngc.collect()\n\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:38.498798Z","iopub.execute_input":"2024-03-23T04:53:38.499778Z","iopub.status.idle":"2024-03-23T04:53:38.638138Z","shell.execute_reply.started":"2024-03-23T04:53:38.499745Z","shell.execute_reply":"2024-03-23T04:53:38.637005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noticed some of these numeric variables were parsed as strings, changed them all to int \nnum_list = ['days120_123L','days180_256L','days30_165L','days360_512L','days90_310L','firstquarter_103L','numinstpaidlastcontr_4325080L',\n            'fourthquarter_440L','numinstlswithdpd5_4187116L','numberofqueries_373L', 'secondquarter_766L','thirdquarter_1082L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')\n\n# Drop more unneeded columns, all based on training missing values, 0 range for numeric, or one unique category\ndrop_list = ['lastapprcommoditytypec_5251766M', 'lastrejectcommodtypec_5251769M','lastrejectcommoditycat_161M','lastrejectreasonclient_4145040M',\n            'previouscontdistrict_112M','lastapprcommoditycat_1041M', 'lastcancelreason_561M','lastrejectreason_759M', 'addres_zip_823M', 'addres_district_368M',\n            'commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L','paytype1st_925L_OTHER','paytype_783L_OTHER','applicationcnt_361L','empls_employer_name_740M_a55475b1']\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:42.978675Z","iopub.execute_input":"2024-03-23T04:53:42.979035Z","iopub.status.idle":"2024-03-23T04:53:42.997321Z","shell.execute_reply.started":"2024-03-23T04:53:42.979006Z","shell.execute_reply":"2024-03-23T04:53:42.996158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Medians and modes after imputation are the same, warning message does not seem to be an issue\n#test = imputer(test)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:46.729244Z","iopub.execute_input":"2024-03-23T04:53:46.729645Z","iopub.status.idle":"2024-03-23T04:53:46.734979Z","shell.execute_reply.started":"2024-03-23T04:53:46.729614Z","shell.execute_reply":"2024-03-23T04:53:46.733691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See boolean columns\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:50.363843Z","iopub.execute_input":"2024-03-23T04:53:50.364555Z","iopub.status.idle":"2024-03-23T04:53:50.374975Z","shell.execute_reply.started":"2024-03-23T04:53:50.364519Z","shell.execute_reply":"2024-03-23T04:53:50.374049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include=['category']).columns.tolist()\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:53:56.639032Z","iopub.execute_input":"2024-03-23T04:53:56.639635Z","iopub.status.idle":"2024-03-23T04:53:56.717911Z","shell.execute_reply.started":"2024-03-23T04:53:56.639605Z","shell.execute_reply":"2024-03-23T04:53:56.716867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install gdown","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:54:09.839136Z","iopub.execute_input":"2024-03-23T04:54:09.839562Z","iopub.status.idle":"2024-03-23T04:54:25.055871Z","shell.execute_reply.started":"2024-03-23T04:54:09.839526Z","shell.execute_reply":"2024-03-23T04:54:25.054493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gdown \nurl = 'http://drive.google.com/uc?id=1vgBhqZJSKFuzfZiEYd6iArgRrHU6KYlZ'\noutput ='train_final_dummy.zip'\ngdown.download(url, output, quiet=False)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:54:28.443951Z","iopub.execute_input":"2024-03-23T04:54:28.444794Z","iopub.status.idle":"2024-03-23T04:54:30.495402Z","shell.execute_reply.started":"2024-03-23T04:54:28.444753Z","shell.execute_reply":"2024-03-23T04:54:30.494309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:54:34.639209Z","iopub.execute_input":"2024-03-23T04:54:34.639597Z","iopub.status.idle":"2024-03-23T04:54:34.645135Z","shell.execute_reply.started":"2024-03-23T04:54:34.63957Z","shell.execute_reply":"2024-03-23T04:54:34.643935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with zipfile.ZipFile(\"train_final_dummy.zip\", \"r\") as file:\n file.extractall(\"train_final_dummy\")","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:54:40.789464Z","iopub.execute_input":"2024-03-23T04:54:40.789833Z","iopub.status.idle":"2024-03-23T04:54:48.659612Z","shell.execute_reply.started":"2024-03-23T04:54:40.789806Z","shell.execute_reply":"2024-03-23T04:54:48.658591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.listdir(\"train_final_dummy\")","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:54:52.554344Z","iopub.execute_input":"2024-03-23T04:54:52.554763Z","iopub.status.idle":"2024-03-23T04:54:52.563208Z","shell.execute_reply.started":"2024-03-23T04:54:52.554733Z","shell.execute_reply":"2024-03-23T04:54:52.562117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:19:19.949765Z","iopub.execute_input":"2024-03-22T03:19:19.95008Z","iopub.status.idle":"2024-03-22T03:19:19.954429Z","shell.execute_reply.started":"2024-03-22T03:19:19.950059Z","shell.execute_reply":"2024-03-22T03:19:19.953127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"train_final_dummy/train_final_dummy.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:19:37.46006Z","iopub.execute_input":"2024-03-22T03:19:37.460393Z","iopub.status.idle":"2024-03-22T03:20:01.348167Z","shell.execute_reply.started":"2024-03-22T03:19:37.460369Z","shell.execute_reply":"2024-03-22T03:20:01.347081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert polars df to pandas so pandas specific methods/attributes work later, seems more memory efficient to load as pl and convert to pd than load as pd\ntrain = train.to_pandas()\n# Get list of ids for submission file\nids = test['case_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:20:36.8201Z","iopub.execute_input":"2024-03-22T03:20:36.820805Z","iopub.status.idle":"2024-03-22T03:20:36.997886Z","shell.execute_reply.started":"2024-03-22T03:20:36.820768Z","shell.execute_reply":"2024-03-22T03:20:36.996459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-03-22T08:02:08.677695Z","iopub.execute_input":"2024-03-22T08:02:08.678173Z","iopub.status.idle":"2024-03-22T08:02:08.684161Z","shell.execute_reply.started":"2024-03-22T08:02:08.678137Z","shell.execute_reply":"2024-03-22T08:02:08.682791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_csv(\"train_final_dummy/train_final_dummy.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:44.639358Z","iopub.execute_input":"2024-03-23T04:56:44.639735Z","iopub.status.idle":"2024-03-23T04:56:53.050375Z","shell.execute_reply.started":"2024-03-23T04:56:44.639705Z","shell.execute_reply":"2024-03-23T04:56:53.049282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert polars df to pandas so pandas specific methods/attributes work later, seems more memory efficient to load as pl and convert to pd than load as pd\ntrain = train.to_pandas()\n# Get list of ids for submission file\nids = test['case_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:39.878743Z","iopub.execute_input":"2024-03-23T04:56:39.879459Z","iopub.status.idle":"2024-03-23T04:56:39.911526Z","shell.execute_reply.started":"2024-03-23T04:56:39.879418Z","shell.execute_reply":"2024-03-23T04:56:39.909913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only select common columns to use \ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest=test[common_columns]\n\n# Subset train with only columns seen in test + target\ntrain = train[common_columns+['target']]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:34.36894Z","iopub.execute_input":"2024-03-23T04:56:34.369927Z","iopub.status.idle":"2024-03-23T04:56:34.402979Z","shell.execute_reply.started":"2024-03-23T04:56:34.369891Z","shell.execute_reply":"2024-03-23T04:56:34.401784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit on stratified sample, 1% sample\n# Note: no random seed, warning message is fine\ntrain_sample = train.groupby('target', group_keys=False).apply(lambda x: x.sample(frac=0.01)).reset_index(drop=True)\ntrain_sample.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:31.568783Z","iopub.execute_input":"2024-03-23T04:56:31.569411Z","iopub.status.idle":"2024-03-23T04:56:31.604085Z","shell.execute_reply.started":"2024-03-23T04:56:31.569379Z","shell.execute_reply":"2024-03-23T04:56:31.60255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_sample.loc[:,'target'].to_frame('target')\nX = train_sample.drop(['target',], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:05.559007Z","iopub.execute_input":"2024-03-23T04:56:05.561533Z","iopub.status.idle":"2024-03-23T04:56:05.597211Z","shell.execute_reply.started":"2024-03-23T04:56:05.561478Z","shell.execute_reply":"2024-03-23T04:56:05.595485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check target distribution\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(),4))","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:01.878244Z","iopub.execute_input":"2024-03-23T04:56:01.87862Z","iopub.status.idle":"2024-03-23T04:56:01.913192Z","shell.execute_reply.started":"2024-03-23T04:56:01.878593Z","shell.execute_reply":"2024-03-23T04:56:01.911859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, train_sample\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-23T04:56:13.528907Z","iopub.execute_input":"2024-03-23T04:56:13.529946Z","iopub.status.idle":"2024-03-23T04:56:13.565982Z","shell.execute_reply.started":"2024-03-23T04:56:13.529909Z","shell.execute_reply":"2024-03-23T04:56:13.564586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numeric_cols = test.select_dtypes(include=['number']).columns.tolist()\n\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nX.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:30:40.267297Z","iopub.execute_input":"2024-03-22T03:30:40.267986Z","iopub.status.idle":"2024-03-22T03:30:40.361567Z","shell.execute_reply.started":"2024-03-22T03:30:40.26795Z","shell.execute_reply":"2024-03-22T03:30:40.360711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:31:05.471406Z","iopub.execute_input":"2024-03-22T03:31:05.471946Z","iopub.status.idle":"2024-03-22T03:31:05.496775Z","shell.execute_reply.started":"2024-03-22T03:31:05.471922Z","shell.execute_reply":"2024-03-22T03:31:05.495915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 80/20 split\nX_train, X_valid, y_train, y_valid= train_test_split(X, y, test_size=0.2, stratify=y, random_state = 123)","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:31:40.457381Z","iopub.execute_input":"2024-03-22T03:31:40.457972Z","iopub.status.idle":"2024-03-22T03:31:40.539973Z","shell.execute_reply.started":"2024-03-22T03:31:40.457941Z","shell.execute_reply":"2024-03-22T03:31:40.539028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:31:53.076514Z","iopub.execute_input":"2024-03-22T03:31:53.076862Z","iopub.status.idle":"2024-03-22T03:31:53.178321Z","shell.execute_reply.started":"2024-03-22T03:31:53.076836Z","shell.execute_reply":"2024-03-22T03:31:53.177132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check proper split\nprint(X_train.shape)\nprint(X_valid.shape)\nprint(y_train.shape)\nprint(y_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:32:11.66719Z","iopub.execute_input":"2024-03-22T03:32:11.66754Z","iopub.status.idle":"2024-03-22T03:32:11.673854Z","shell.execute_reply.started":"2024-03-22T03:32:11.667514Z","shell.execute_reply":"2024-03-22T03:32:11.672107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nwarnings.filterwarnings(\"ignore\")\nprint(\"Before Undersampling, counts of label '1': {}\".format(sum(y_train['target'] == 1))) \nprint(\"Before Undersampling, counts of label '0': {} \\n\".format(sum(y_train['target'] == 0)))\n\n# Version 1: Default, selects samples of the majority class for which average distances to the k closest instances of the minority class is smallest\n# Version 2: Selects samples of the majority class for which average distances to the k farthest instances of the minority class is smallest.\n# Version 3: Firstly, for each minority class instance, their M nearest-neighbors will be stored. Then finally, the majority class instances are selected for which the average distance to the N nearest-neighbors is the largest.\nnr = NearMiss(sampling_strategy=0.25)\n\n\nX_train_miss, y_train_miss = nr.fit_resample(X_train, y_train)\nprint('After Undersampling, the shape of train_X: {}'.format(X_train_miss.shape)) \nprint('After Undersampling, the shape of train_y: {} \\n'.format(y_train_miss.shape)) \n  \nprint(\"After Undersampling, counts of label '1': {}\".format(sum(y_train_miss['target'] == 1))) \nprint(\"After Undersampling, counts of label '0': {}\".format(sum(y_train_miss['target'] == 0)))\nwarnings.filterwarnings(\"default\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:32:41.658021Z","iopub.execute_input":"2024-03-22T03:32:41.658382Z","iopub.status.idle":"2024-03-22T03:32:41.665499Z","shell.execute_reply.started":"2024-03-22T03:32:41.658354Z","shell.execute_reply":"2024-03-22T03:32:41.664256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nwarnings.filterwarnings(\"ignore\")\nprint(\"Before Oversampling, counts of label '1': {}\".format(sum(y_train['target'] == 1))) \nprint(\"Before Oversampling, counts of label '0': {} \\n\".format(sum(y_train['target'] == 0)))\n\nsmote = SMOTE(sampling_strategy = 0.25, random_state=123)\n\nX_train_smote, y_train_smote = smote.fit_resample(X_train, y_train)\nprint('After Oversampling, the shape of train_X: {}'.format(X_train_smote.shape)) \nprint('After Oversampling, the shape of train_y: {} \\n'.format(y_train_smote.shape)) \n  \nprint(\"After Oversampling, counts of label '1': {}\".format(sum(y_train_smote['target'] == 1))) \nprint(\"After Oversampling, counts of label '0': {}\".format(sum(y_train_smote['target'] == 0))) \nwarnings.filterwarnings(\"default\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:33:12.597378Z","iopub.execute_input":"2024-03-22T03:33:12.597746Z","iopub.status.idle":"2024-03-22T03:33:12.60513Z","shell.execute_reply.started":"2024-03-22T03:33:12.597717Z","shell.execute_reply":"2024-03-22T03:33:12.603932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The “balanced” mode uses the values of y to automatically adjust weights inversely proportional \n# to class frequencies in the input data as n_samples / (n_classes * np.bincount(y)).\nlr = LogisticRegression(class_weight='balanced', random_state=123)\nlr_params = {\n    'solver': ('lbfgs','newton-cg','newton-cholesky','sag'), # these 4 solvers either use l2 or None penalty\n    'penalty': ('l2', None), 'C': (0.01, 0.1, 1, 10), # Regularization parameter, default = 1.0\n}\n\n# Using roc_auc instead of default accuracy to score due to imbalanced target\n# Competition evaluation metric is based off AUC\nlr_search = GridSearchCV(lr, lr_params, verbose=1, scoring='roc_auc', cv=3)\nprint(\"Hyperparameters to tune are:\")\npprint(lr_params)","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:33:50.566756Z","iopub.execute_input":"2024-03-22T03:33:50.56713Z","iopub.status.idle":"2024-03-22T03:33:50.574716Z","shell.execute_reply.started":"2024-03-22T03:33:50.567103Z","shell.execute_reply":"2024-03-22T03:33:50.573239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\nlr_search.fit(X_train, y_train)\n\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:34:13.047601Z","iopub.execute_input":"2024-03-22T03:34:13.047991Z","iopub.status.idle":"2024-03-22T03:37:14.615902Z","shell.execute_reply.started":"2024-03-22T03:34:13.047964Z","shell.execute_reply":"2024-03-22T03:37:14.615213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Best score= {lr_search.best_score_:0.3f}\")\nprint(\"Best parameters set:\")\nbest_parameters = lr_search.best_estimator_.get_params()\nfor name in sorted(lr_params.keys()):\n    print(\"\\t%s: %r\" % (name, best_parameters[name]))","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:40:52.906838Z","iopub.execute_input":"2024-03-22T03:40:52.908504Z","iopub.status.idle":"2024-03-22T03:40:52.91794Z","shell.execute_reply.started":"2024-03-22T03:40:52.908435Z","shell.execute_reply":"2024-03-22T03:40:52.916048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\ny_pred = lr_search.predict(X_valid)\n\nprint(f'Validation Target: {round(y_valid.target.value_counts()[1]/y_valid.target.value_counts().sum(),4)}')\nprint(f'Validation Accuracy: {accuracy_score(y_valid,y_pred)}')\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:41:24.017418Z","iopub.execute_input":"2024-03-22T03:41:24.017931Z","iopub.status.idle":"2024-03-22T03:41:24.043747Z","shell.execute_reply.started":"2024-03-22T03:41:24.0179Z","shell.execute_reply":"2024-03-22T03:41:24.04275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n# Print confusion matrix with percent\ntarget_names= ['No Default', 'Default']\nmatrix = confusion_matrix(y_valid, y_pred, normalize='true')\ncm_display = ConfusionMatrixDisplay(confusion_matrix= matrix, display_labels=target_names).plot()\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:41:45.330908Z","iopub.execute_input":"2024-03-22T03:41:45.33132Z","iopub.status.idle":"2024-03-22T03:41:45.6166Z","shell.execute_reply.started":"2024-03-22T03:41:45.331289Z","shell.execute_reply":"2024-03-22T03:41:45.615292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n# Print out the report\nprint(classification_report(y_valid, y_pred, target_names = target_names))\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:42:08.271156Z","iopub.execute_input":"2024-03-22T03:42:08.272171Z","iopub.status.idle":"2024-03-22T03:42:08.290551Z","shell.execute_reply.started":"2024-03-22T03:42:08.272126Z","shell.execute_reply":"2024-03-22T03:42:08.289583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best model params\nlr_model1 = lr_search.best_estimator_\nprint(lr_model1)","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:42:40.450974Z","iopub.execute_input":"2024-03-22T03:42:40.451315Z","iopub.status.idle":"2024-03-22T03:42:40.461923Z","shell.execute_reply.started":"2024-03-22T03:42:40.451292Z","shell.execute_reply":"2024-03-22T03:42:40.460494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot ROC AUC curve using skplt\nwarnings.filterwarnings(\"ignore\")\ny_prob = lr_search.predict_proba(X_valid)\nskplt.metrics.plot_roc_curve(y_valid, y_prob)\nplt.show()\nwarnings.filterwarnings('default')","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:43:03.85076Z","iopub.execute_input":"2024-03-22T03:43:03.851092Z","iopub.status.idle":"2024-03-22T03:43:04.113341Z","shell.execute_reply.started":"2024-03-22T03:43:03.851069Z","shell.execute_reply":"2024-03-22T03:43:04.112027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = lr_model1.predict_proba(test)\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:43:55.520297Z","iopub.execute_input":"2024-03-22T03:43:55.520614Z","iopub.status.idle":"2024-03-22T03:43:55.53674Z","shell.execute_reply.started":"2024-03-22T03:43:55.520592Z","shell.execute_reply":"2024-03-22T03:43:55.535966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id': ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-03-22T03:44:33.450635Z","iopub.execute_input":"2024-03-22T03:44:33.451331Z","iopub.status.idle":"2024-03-22T03:44:33.46512Z","shell.execute_reply.started":"2024-03-22T03:44:33.451293Z","shell.execute_reply":"2024-03-22T03:44:33.463987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-03-22T07:51:53.497896Z","iopub.execute_input":"2024-03-22T07:51:53.498954Z","iopub.status.idle":"2024-03-22T07:51:56.08024Z","shell.execute_reply.started":"2024-03-22T07:51:53.498903Z","shell.execute_reply":"2024-03-22T07:51:56.079088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Assuming train_data is a DataFrame containing your training data\ntrain_data = pd.read_csv(\"train_final_dummy/train_final_dummy.csv\")\n\n# Set the figure size\nsns.set(rc={'figure.figsize': (20, 16)})\n\n# Plot the histogram\ntrain_data.hist(color='mediumseagreen')\n\n# Adjust layout\nplt.tight_layout()\n\n# Add title\nplt.suptitle('Feature Distributions', y=1.02, fontsize=20)\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T07:58:53.987817Z","iopub.execute_input":"2024-03-22T07:58:53.988295Z","iopub.status.idle":"2024-03-22T07:58:54.455612Z","shell.execute_reply.started":"2024-03-22T07:58:53.988263Z","shell.execute_reply":"2024-03-22T07:58:54.454221Z"},"trusted":true},"execution_count":null,"outputs":[]}]}