{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8274557,"sourceType":"datasetVersion","datasetId":4506020}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is our Final Capstone Project ","metadata":{}},{"cell_type":"markdown","source":"### Step 1:  Feature Engineering and Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# add all the necessary libraries\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nimport lightgbm as lgb\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, ConfusionMatrixDisplay, classification_report, RocCurveDisplay\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold\nfrom contextlib import suppress\nimport catboost\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here I will make a set of User Defined Functions that will help with data preprocessing:","metadata":{}},{"cell_type":"code","source":"# this function takes a data frame as input, and iterates over all the columns\n#each column ending with a P or an A, it casts the column to Float 64\ndef set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        #Transform DPD (days past due) and A (amount) as Float64 type\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        #cast aggregated numeric columns as Float64 type\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\n#this function takes a data frame as input, and iterates over all the columns\n#each columns that is a string or an object...it converts the col to string type, then to cat type\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n#this function prints out the percent of missing values\n\ndef missing_values(df, threshold = 0.9):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:\n            print(f\"{col}: {decimal}\")\n            \n#imputes numeric columns with the median and cat with mode\n#mode_without_nan represents the most frequent value in the column, exluding any missing values\n#dropna() removes missing values NAN from the column, and returns a new series with non-null values\n# .mode() calculates the mode in the series\n# .values[0] extracts the first (or only) mode from .mode(), which is a series (we are accessing first element)\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype in ['float64']:\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category', 'object'] and df[col].isull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(model_without_nan)\n    return df\n\n\n#this takes a series as input and calculates the range (difference between max and min values in series)\ndef custom_range_agg(series: pl.series):\n    custom_range = series.max() - series.min()\n    return custom_range\n\n\n\n# this function is used to compare sum and range aggregated features against each other, printing the percent of the range column that either equals 0 or the value of the sum column\n#drops range feature if percent > 0.9.  \n# this function will also replace nulls with 0s only for range columns\n#filters out aggregated range columns that provide little information\n\ndef compare_cols(df):\n    range_list = [element for e in df.columns for element in e.split() if element.endswith(\"_range\")]\n    sum_list = [name.replace('range', 'sum') for name in range_list]\n    length = len(df)\n    \n    df=df.with_columns(*[pl.col(col).fill_null(strategy='zero') for col in range_list])\n    \n    for i in range(len(range_list)):\n        df = df.with_columns(check=pl.when((df[range_list[i]] == df[sum_list[i]]) | (df[range_list[i]] == 0)).then(1).otherwise(0))\n        percent = df.select(pl.sum(\"check\")).item() / length\n        print(f\"Percent equal or 0 for {range_list[i]} = {percent:.3f}\")\n        \n    df = df.drop(\"check\")\n    return df\n\n\n# this will reduce memory usage of dataframes.  It casts all numeric columns to lowest memory datatype given column range\ndef reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n            \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n                    \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n                    \n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate Joined Test Data","metadata":{}},{"cell_type":"markdown","source":"### Part 1","metadata":{}},{"cell_type":"markdown","source":"We preprocessed the training files in the same way.  The lsits of columns that are dropped were manually taken from what was dropped in the training based on excessive missing values.  We used .drop(errors='ignore') to handle the situations where the hidden test set had different columns.","metadata":{}},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway + \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# additional depth = 1 files\n\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this code cell copied from Kevin\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# use aggregation functions in tables with depth >=1\n#this code cell commented out because error somewhere\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who originally applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\", \"empl_employedfrom_271D\", \"empl_industry_691L\", \"familystate_447L\", \"sex_738L\", \"type_25L\",\n                                             \"safeguarantyflag_411L\", \"empl_employedtotal_800L\", \"role_1084L\"]).filter(pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))\n\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this copied from Kevin\n# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"), \n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# additional aggregation for depth = 1 files\n#commented this out because error\n\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositougoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"),\n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))\n    \n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n#after merge, convert back to pandas for errors = 'ignore' functionality\n\njoin_data1 = join_data1.to_pandas()\n\ndrop_cols = ['clientscnt_136L', 'datelastinstal40dpd_247D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'interestrategrace_34L', 'isbidproductrequest_292L', 'lastdependentsnum_448L',\n            'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'payvacationpostpone_4187118D', 'validfrom_1069D', 'assignmentdate_238D',\n            'assignmentdate_4527235D', 'assignmentdate_4955616D', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L',\n            'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A',\n            'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'riskassessment_302T', 'riskassessment_940T', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum',\n            'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum',\n            'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum',\n            'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum',\n            'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\n\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors=\"ignore\")\njoin_data1.shape\n\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#copied this next cell from Kevin's notebook","metadata":{}},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n# Based on training preprocessing, shoudl be 194 - 1 = 193 cols\ndrop_cols = ['clientscnt_136L', 'datelastinstal40dpd_247D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'interestrategrace_34L', 'isbidproductrequest_292L', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'payvacationpostpone_4187118D', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'riskassesment_302T', 'riskassesment_940T', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1, test_credit_bureau_b_1, test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 2","metadata":{}},{"cell_type":"code","source":"#Kevin's notebook\n\n# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\nselection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\ntest_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    custom_range_agg(pl.col(\"annuity_853A\")).alias('annuity_853A_range'),\n    pl.col(\"byoccupationinc_3656910L\").sum().alias(\"byoccupationinc_3656910L_sum\"),\n    custom_range_agg(pl.col(\"byoccupationinc_3656910L\")).alias('byoccupationinc_3656910L_range'),\n    pl.col(\"childnum_21L\").sum().alias(\"childnum_21L_sum\"),\n    custom_range_agg(pl.col(\"childnum_21L\")).alias('childnum_21L_range'),\n    pl.col(\"credacc_credlmt_575A\").sum().alias(\"credacc_credlmt_575A_sum\"),\n    custom_range_agg(pl.col(\"credacc_credlmt_575A\")).alias('credacc_credlmt_575A_range'),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    custom_range_agg(pl.col(\"downpmt_134A\")).alias('downpmt_134A_range'),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    custom_range_agg(pl.col(\"mainoccupationinc_437A\")).alias('mainoccupationinc_437A_range'),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    custom_range_agg(pl.col(\"pmtnum_8L\")).alias('pmtnum_8L_range'),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"),\n    custom_range_agg(pl.col(\"tenor_203L\")).alias('tenor_203L_range'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\ntest_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"),\n    custom_range_agg(pl.col(\"amount_4527230A\")).alias('amount_4527230A_range'))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = compare_cols(test_applprev_1_feats_1)\ntest_tax_registry_a_1_feats = compare_cols(test_tax_registry_a_1_feats)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\ntest_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    custom_range_agg(pl.col(\"monthlyinstlamount_332A\")).alias('monthlyinstlamount_332A_range'),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    custom_range_agg(pl.col(\"monthlyinstlamount_674A\")).alias('monthlyinstlamount_674A_range'),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    custom_range_agg(pl.col(\"numberofinstls_229L\")).alias('numberofinstls_229L_range'),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    custom_range_agg(pl.col(\"numberofoverdueinstlmax_1151L\")).alias('numberofoverdueinstlmax_1151L_range'),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"),\n    custom_range_agg(pl.col(\"totalamount_6A\")).alias('totalamount_6A_range'))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))\n\ntest_credit_bureau_a_1_feats = compare_cols(test_credit_bureau_a_1_feats)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\njoin_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','amount_4917619A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1, test_credit_bureau_a_1\ndel test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Part 3","metadata":{}},{"cell_type":"code","source":"#Additional testing data, depth=2\n\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\nsel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes),\n\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\ntest_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_1073P\")).alias('pmts_dpd_1073P_range'),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    custom_range_agg(pl.col(\"pmts_dpd_303P\")).alias('pmts_dpd_303P_range'),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1140A\")).alias('pmts_overdue_1140A_range'),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"),\n    custom_range_agg(pl.col(\"pmts_overdue_1152A\")).alias('pmts_overdue_1152A_range'))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_2_feats = compare_cols(test_credit_bureau_a_2_feats)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\njoin_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\ndel test_applprev_2, test_person_2, test_credit_bureau_a_2\ndel test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n#convert back to polars for datetime\njoin_test = pl.from_pandas(join_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Join Test Data","metadata":{}},{"cell_type":"code","source":"#Kevin's notebook\n\ntest = join_test.with_columns(pl.col('date_decision').cast(pl.Date))\n# Feature engineer date columns\ndate_list = ['datefirstoffer_1144D', 'datelastunpaid_3546854D', 'dtlastpmtallstes_4499206D', 'firstclxcampaign_1125D', 'lastdelinqdate_224D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'birthdate_574D', 'responsedate_1012D', 'responsedate_4527233D', 'responsedate_4917613D', 'empl_employedfrom_271D',\n            'firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D','birth_259D']\n\n# Calculates date_diff as date_decision minus each date column from above\nfor col in date_list:\n    test = test.with_columns(pl.col(col).cast(pl.Date))\n    test = test.with_columns(\n        ((pl.col(\"date_decision\") - pl.col(col)) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(f\"{col}_diff\"))\n\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\n# Convert _range features to Float64\nfor col in test.columns:\n        if col[-5:-1] in (\"range\"):\n            test = test.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n            \ndrop_list = ['date_decision','MONTH'] + date_list\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(drop_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test, dfs\ngc.collect()\n\ntest.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\n# Noticed these numeric variables were parsed as strings, changed them all to float (same as training preprocessing)\nnum_list = ['numinstlswithdpd5_4187116L', 'numinstmatpaidtearly2d_4499204L','numinstpaid_4499208L','numinstpaidearly3dest_4493216L','numinstpaidearly5dest_4493211L',\n           'numinstpaidearly5dobd_4499205L','numinstpaidearlyest_4493214L','numinstpaidlastcontr_4325080L','numinstregularpaidest_4493210L','numinsttopaygrest_4493213L','numinstunpaidmaxest_4493212L',\n           'contractssum_5085716L','days120_123L','days180_256L','days360_512L','firstquarter_103L','fourthquarter_440L','numberofqueries_373L','pmtscount_423L','secondquarter_766L','thirdquarter_1082L',\n           'days30_165L','days90_310L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Kevin's notebook\n\n# Same as training preprocessing (features either all the same or categorical variables with unique categories > 150)\ndrop_list = ['commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L','addres_zip_823M',\n            'lastapprcommoditytypec_5251766M','lastrejectcommodtypec_5251769M','previouscontdistrict_112M', 'addres_district_368M']\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\n# Drop rows with all na\ntest.dropna(axis=1, how='all', inplace=True)\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imputation","metadata":{}},{"cell_type":"code","source":"#see boolean columns\n\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# convert boolean columns to 0 or 1 (false or true)\n\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n#check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.count_nonzero(test.isnull()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#verify the data types are correct after imputation\n\npd.set_option('display.max_rows', None)\ntest.dtypes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n\n#here is how you would create dummy variables, but commented out since we did CatBoost model\n\ncat_cols = test.select_dtypes(include=['category', 'object']).columns.tolist()\n#create dummies for all cat columns, not dropping first col to keep column names the same as training set\n\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()\n\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = reduce_mem_usage(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model\n","metadata":{}},{"cell_type":"code","source":"#This reads the csv file into the dataframe, and applies to UDAs and converts columns that end with  'range'\n #to Float64\ntrain = pl.read_csv('/kaggle/input/training/train251_noimpute.csv').pipe(set_table_dtypes).pipe(convert_strings)\n\nfor col in train.columns:\n    if col[-5:-1] in (\"range\"):\n        train = train.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        \ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert polars df to pandas so pandas stuff works, and more memory efficient to load as pl and convert to pandas\n\ntrain = train.to_pandas()\n\n#get list of ids for submission file\nids = test['case_id'].tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#apply our UDA to the dataframe\ntrain = reduce_mem_usage(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# only select common columns to use\n\ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest=test[common_columns]\n\n#subset train with only columns seen in test + target\n\ntrain = train[common_columns + ['target']]\ntrain.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create a data frame y that contains only the 'target' column from the train data frame\n#create a data frame X that drops the 'target' column from the train data frame\n\ny = train.loc[:, 'target'].to_frame('target')\nX = train.drop(['target'], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check target distribution\n\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(), 4))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#do not include case_id, or week_num as numeric variable\n\nnumeric_cols = test.select_dtypes(include=['number']).columns.tolist()\nnumeric_cols.remove('case_id')\nnumeric_cols.remove('WEEK_NUM')\nprint(numeric_cols)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#normalize the numeric variables\n\nwarnings.filterwarnings(\"ignore\")\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nwarnings.filterwarnings(\"default\")\n\nX.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"code","source":"# drop case_id and week_num from features\n\nweeks = X[\"WEEK_NUM\"]\n\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n# sort columns in alphabetical order so they match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\n\nprint(X_feats.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include = ['category', 'object']).columns.tolist()\nprint(cat_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_feats[cat_cols] = X_feats[cat_cols].astype(str)\ntest[cat_cols] = test[cat_cols].astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"#set up a cross-validation process using 'StratifiedGroupKFold'\n#define parameters for LGBM model\n#each fold contains approximately the same distribution of target classes while also preserving group structures\n\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\n#store the trained models obtained curing cross validation\nfitted_models_cat = []\nfitted_models_lgb = []\n\n#store the cross validation scores\ncv_scores_cat = []\ncv_scores_lgb = []\n\n#running with GPU\n\ngrid_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.06,\n    \"n_estimators\": 2500,\n    \"colsample_bytree\": 0.8,\n    \"colslample_bynode\": 0.8,\n    \"random_state\": 42,\n    \"reg_alpha\": 10,\n    \"extra_trees\": True,\n    \"num_leaves\": 64,\n    \"verbose\": -1,\n    'device' : 'gpu',\n    \n}\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\n\n#ensemble model inspired by other notebooks\n#group by weeks to account for gini instability weekly metric\n#performs a cross_validation training of two different models, CatBoost and LightGBM\n#for each fold, it splits the data into training and validation sets\n#fits both models to training data, and then calculates AUC score on validation set and appends both models and AUC score to respective list\n\nfor idx_train, idx_valid in cv.split(X_feats, y, groups=weeks):\n    #split training in ever fold by setting the split indexes\n    X_train, y_train = X_feats.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X_feats.iloc[idx_valid], y.iloc[idx_valid]\n    #pool used for CatBoost\n    train_pool = Pool(X_train, y_train, cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid, cat_features=cat_cols)\n    \n    clf = CatBoostClassifier(\n    eval_metric='AUC', task_type = 'GPU',\n    learning_rate=0.05, iterations=5000,\n    random_seed=3107)\n    clf.fit(train_pool, eval_set=val_pool, verbose=300)\n    #append model and auc_score to their appropriate list\n    fitted_models_cat.append(clf)\n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n    \n    X_train[cat_cols] = X_train[cat_cols].astype(\"category\")\n    X_valid[cat_cols] = X_valid[cat_cols].astype(\"category\")\n    \n    #train lgbm model, stopping if no improvement in auc in 200 rounds\n    model = lgb.LGBMClassifier(**grid_params)\n    model.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(200)])\n    #again, append model and auc\n    fitted_models_lgb.append(model)\n    y_pred_valid = model.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_lgb.append(auc_score)\n    \n    \n    del clf, model\n    gc.collect()\n    \nprint(\"CV CAT AUC scores: \", cv_scores_cat)\nprint(\"Maximum CV CAT AUC score: \", max(cv_scores_cat))\n\nprint(\"CV LGB AUC scores: \", cv_scores_lgb)\nprint(\"Maximum CV LGB AUC score: \", max(cv_scores_lgb))\n\nwarnings.filterwarnings(\"default\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set the best cv model\n#defines a VotingModel class, which combines predictions from multiple models\n\nclass VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n    \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators[:5]]\n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        y_preds += [estimator.predict(X) for estimator in self.estimators[5:10]]\n        # can test including or not, would give more weight to CatBoost and LGBM predicts\n        \n        return np.mean(y_preds, axis=0)\n    \n    # main use case for this class\n    def predict_proba(self, X):\n        #predict_proba for CatBoost\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]\n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        #predict_proba for LGBM\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[5:10]]\n        #can test including or not, would give more weight to CatBoost and LGBM predicts\n        \n        return np.mean(y_preds, axis=0)\n    \nmodel = VotingModel(fitted_models_cat + fitted_models_lgb)\nmodel_lgb = fitted_models_lgb[np.argmax(cv_scores_lgb)]\nmodel_cat = fitted_models_cat[np.argmax(cv_scores_cat)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del fitted_models_cat, fitted_models_lgb\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#K\nwarnings.filterwarnings(\"ignore\")\ny_pred = model.predict(X_feats)\ny_pred1 = [1 if num > 0.50 else 0 for num in y_pred]\n\ntarget_names= ['No Default', 'Default']\nmatrix = confusion_matrix(y, y_pred1, normalize='true')\ncm_display = ConfusionMatrixDisplay(confusion_matrix= matrix, display_labels=target_names).plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion Matrix and Classification Report","metadata":{}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\ny_pred = model.predict(X_feats)\ny_pred1 = [1 if num > 0.50 else 0 for num in y_pred]\n\ntarget_names = ['No Default', 'Default']\nmatrix = confusion_matrix(y, y_pred1, normalize='true')\ncm_display = ConfusionMatrixDisplay(confusion_matrix = matrix, display_labels = target_names).plot","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print out the report\nprint(classification_report(y, y_pred1, target_names = target_names))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot the ROC AUC curve for all training data\n\nfpr, tpr, threshold = metrics.roc_curve(y_valid, y_pred_valid)\ndisplay = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc = auc_score)\ndisplay.plot()\nplt.plot([0,1], [0,1], 'r--')\nplt.title('ROC-AUC Curve for Validation Data')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_pred, y_pred1\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Importance","metadata":{}},{"cell_type":"code","source":"#for lgb model, ranks features based on number of splits\n\nlgb.plot_importance(model_lgb, figsize=(20, 20), max_num_features = 20)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for lgb model, ranks on the gain\n# gain is the relative contribution of each feature to the model.\n# total gain is just the sum of gains across all splits\n\nlgb.plot_importance(model_lgb, figsize=(20, 20), importance_type = 'gain', max_num_features = 20)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_feature_importance(importance,columns,model_name):\n    #create arrays for feature importance and feature names\n    feature_importance = np.array(importance)\n    feature_names = np.array(columns)\n    \n    #create a df using a dictionary\n    data = {'feature_names': feature_names, 'feature_importance': feature_importance}\n    temp_df = pd.DataFrame(data)\n    \n    #sort the df in order of decreasing feature importance\n    temp_df.sort_values(by=['feature_importance'], ascending=False, inplace=True)\n    \n    plt.figure(figsize=(20, 20))\n    #plot only top 20 features\n    sns.barplot(x=temp_df['feature_importance'][:20], y=temp_df['feature_names'][:20])\n    \n    plt.title(model_name)\n    plt.xlabel('Feature Importance')\n    plt.ylabel('Feature Names')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculates the importance of each feature by comparing the model's performance with and without the variable\n#difference in the loss function obtained when including the feature versus exluding it.  The bigger the difference, the more important it is.\n\nplot_feature_importance(model_cat.get_feature_importance(type='PredictionValuesChange'), X_train.columns, \"CATBOOST\")\nwarnings.filterwarnings('default')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_feats, X_train, X_valid, y_train, y_valid, y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# sort columns alphabetically to match the loaded model\n\ntest = test.reindex(sorted(test.columns), axis=1)\n\npredictions = model.predict_proba(test.drop(['case_id', 'WEEK_NUM'], axis=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id': ids, 'score': predictions[:, 1]}).set_index('case_id')\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}