{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8093776,"sourceType":"datasetVersion","datasetId":4506020}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Home Credit Model and Submission","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings, os, gc, joblib\nfrom pprint import pprint\nimport lightgbm as lgb\nfrom sklearn import metrics\nfrom functools import reduce\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, ConfusionMatrixDisplay, classification_report\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV, StratifiedGroupKFold\nfrom contextlib import suppress","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:00.806389Z","iopub.execute_input":"2024-04-11T18:13:00.806832Z","iopub.status.idle":"2024-04-11T18:13:03.081615Z","shell.execute_reply.started":"2024-04-11T18:13:00.806768Z","shell.execute_reply":"2024-04-11T18:13:03.079885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pathway = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\ndef set_table_dtypes(df: pl.DataFrame)-> pl.DataFrame:\n    for col in df.columns:\n        # Cast Transform DPD (Days past due, P) and Transform Amount (A) as Float64\n        if col[-1] in (\"P\", \"A\"):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        # Cast Transform date (D) as Date, causes issues with other columns ending in D\n        #if col[-1] in (\"D\"):\n            #df = df.with_columns(pl.col(col).cast(pl.Date).alias(col))\n        # Cast aggregated columns as Float64, tried combining sum and max, but did not work correctly\n        if col[-4:-1] in ('_sum'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n        if col[-4:-1] in ('_max'):\n            df = df.with_columns(pl.col(col).cast(pl.Float64).alias(col))\n    return df\n\ndef convert_strings(df: pl.DataFrame) -> pl.DataFrame:\n    for col in df.columns:\n        if df[col].dtype == pl.Utf8:\n            df = df.with_columns(pl.col(col).cast(pl.Categorical))\n    return df\n\n# Changed this function to work for Pandas\ndef missing_values(df, threshold = 0.0):\n    for col in df.columns:\n        decimal = (pd.isnull(test[col]).sum())/(len(test[col]))\n        if decimal > threshold:                                         \n            print(f\"{col}: {decimal}\")\n\n# Impute numeric columns with the median and cat with mode\ndef imputer(df:pd.DataFrame) -> pd.DataFrame:\n    for col in df.columns:\n        if df[col].dtype in ['float64']:\n            df[col] = df[col].fillna(df[col].median())\n        if df[col].dtype.name in ['category','object'] and df[col].isnull().any():\n            mode_without_nan = df[col].dropna().mode().values[0]\n            df[col] = df[col].fillna(mode_without_nan)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.084636Z","iopub.execute_input":"2024-04-11T18:13:03.085507Z","iopub.status.idle":"2024-04-11T18:13:03.104695Z","shell.execute_reply.started":"2024-04-11T18:13:03.085459Z","shell.execute_reply":"2024-04-11T18:13:03.103144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taken from other competition notebooks\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.106523Z","iopub.execute_input":"2024-04-11T18:13:03.106981Z","iopub.status.idle":"2024-04-11T18:13:03.126531Z","shell.execute_reply.started":"2024-04-11T18:13:03.106949Z","shell.execute_reply":"2024-04-11T18:13:03.125040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate joined test data \n#### Part 1\nPreprocessed the training csvs in the same way, lists of columns that are dropped were manually taken from what was dropped in training based on excessive missing values. Used .drop(errors='ignore') to handle situations where hidden test set has different columns. ","metadata":{}},{"cell_type":"code","source":"test_basetable = pl.read_csv(pathway + \"csv_files/test/test_base.csv\")\ntest_static = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_static_0_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_static_0_2.csv\").pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\ntest_static_cb=pl.read_csv(pathway + \"csv_files/test/test_static_cb_0.csv\").pipe(set_table_dtypes)\ntest_person_1=pl.read_csv(pathway +  \"csv_files/test/test_person_1.csv\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2=pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_2.csv\").pipe(set_table_dtypes)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.131102Z","iopub.execute_input":"2024-04-11T18:13:03.131817Z","iopub.status.idle":"2024-04-11T18:13:03.335069Z","shell.execute_reply.started":"2024-04-11T18:13:03.131728Z","shell.execute_reply":"2024-04-11T18:13:03.333649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional depth=1 files\ntest_other_1 = pl.read_csv(pathway + \"csv_files/test/test_other_1.csv\").pipe(set_table_dtypes)\n\ntest_credit_bureau_b_1 = pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n\ntest_deposit_1 = pl.read_csv(pathway + \"csv_files/test/test_deposit_1.csv\").pipe(set_table_dtypes)\n\n# test_debitcard_1 = pl.read_csv(pathway + \"csv_files/test/test_debitcard_1.csv\").pipe(set_table_dtypes)\ntest_basetable = test_basetable.with_columns(pl.col('date_decision').cast(pl.Date))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.336427Z","iopub.execute_input":"2024-04-11T18:13:03.337689Z","iopub.status.idle":"2024-04-11T18:13:03.377389Z","shell.execute_reply.started":"2024-04-11T18:13:03.337644Z","shell.execute_reply":"2024-04-11T18:13:03.376006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#use aggregation functions in tables with depth >=1\n\ntest_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").sum().alias(\"mainoccupationinc_384A_sum\"))\n\n#num_group1=0 represents the person who applied for the loan\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"incometype_1044T\", \"birth_259D\",\n    \"empl_employedfrom_271D\",\"empl_industry_691L\",\"familystate_447L\",\"sex_738L\",\"type_25L\",\n    \"safeguarantyflag_411L\",\"empl_employedtotal_800L\",\"role_1084L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\n#we now have num_group1 and num_group2, so aggregate again\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").sum().alias(\"pmts_pmtsoverdue_635A_sum\"),\n    pl.col(\"pmts_dpdvalue_108P\").sum().alias(\"pmts_dpdvalue_108P_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.379688Z","iopub.execute_input":"2024-04-11T18:13:03.380236Z","iopub.status.idle":"2024-04-11T18:13:03.412940Z","shell.execute_reply.started":"2024-04-11T18:13:03.380192Z","shell.execute_reply":"2024-04-11T18:13:03.411539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Additional aggregation for depth=1 files\ntest_other_1_feats = test_other_1.group_by(\"case_id\").agg(\n    pl.col(\"amtdebitincoming_4809443A\").sum().alias(\"amtdebitincoming_4809443A_sum\"),\n    pl.col(\"amtdebitoutgoing_4809440A\").sum().alias(\"amtdebitoutgoing_4809440A_sum\"),\n    pl.col(\"amtdepositbalance_4809441A\").sum().alias(\"amtdepositbalance_4809441A_sum\"),\n    pl.col(\"amtdepositincoming_4809444A\").sum().alias(\"amtdepositincoming_4809444A_sum\"),\n    pl.col(\"amtdepositoutgoing_4809442A\").sum().alias(\"amtdepositoutgoing_4809442A_sum\"))\n\ntest_credit_bureau_b_1_feats = test_credit_bureau_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_1115A\").sum().alias(\"amount_1115A_sum\"),\n    pl.col(\"credquantity_1099L\").sum().alias(\"credquantity_1099L_sum\"),\n    pl.col(\"credquantity_984L\").sum().alias(\"credquantity_984L_sum\"),\n    pl.col(\"debtpastduevalue_732A\").sum().alias(\"debtpastduevalue_732A_sum\"),\n    pl.col(\"debtvalue_227A\").sum().alias(\"debtvalue_227A_sum\"),\n    pl.col(\"dpd_550P\").sum().alias(\"dpd_550P_sum\"),\n    pl.col(\"dpd_733P\").sum().alias(\"dpd_733P_sum\"),\n    pl.col(\"dpdmax_851P\").max().alias(\"dpdmax_851P_max\"),\n    pl.col(\"installmentamount_644A\").sum().alias(\"installmentamount_644A_sum\"),\n    pl.col(\"installmentamount_833A\").sum().alias(\"installmentamount_833A_sum\"),\n    pl.col(\"instlamount_892A\").sum().alias(\"instlamount_892A_sum\"),\n    pl.col(\"interestrateyearly_538L\").max().alias(\"interestrateyearly_538L_max\"),\n    pl.col(\"maxdebtpduevalodued_3940955A\").max().alias(\"maxdebtpduevalodued_3940955A_max\"),\n    pl.col(\"numberofinstls_810L\").sum().alias(\"numberofinstls_810L_sum\"),\n    pl.col(\"overdueamountmax_950A\").max().alias(\"overdueamountmax_950A_max\"),\n    pl.col(\"pmtdaysoverdue_1135P\").sum().alias(\"pmtdaysoverdue_1135P_sum\"),\n    pl.col(\"pmtnumpending_403L\").sum().alias(\"pmtnumpending_403L_sum\"),\n    pl.col(\"residualamount_3940956A\").sum().alias(\"residualamount_3940956A_sum\"),\n    pl.col(\"totalamount_503A\").sum().alias(\"totalamount_503A_sum\"),\n    pl.col(\"totalamount_881A\").sum().alias(\"totalamount_881A_sum\"))\n\ntest_deposit_1_feats = test_deposit_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_416A\").sum().alias(\"amount_416A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.415209Z","iopub.execute_input":"2024-04-11T18:13:03.415852Z","iopub.status.idle":"2024-04-11T18:13:03.439306Z","shell.execute_reply.started":"2024-04-11T18:13:03.415805Z","shell.execute_reply":"2024-04-11T18:13:03.438328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all tables/columns together\n\njoin_data1 = test_basetable.join(test_static, how=\"left\", on=\"case_id\"\n).join(test_static_cb, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(test_other_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_deposit_1_feats, how=\"left\", on=\"case_id\"\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.440944Z","iopub.execute_input":"2024-04-11T18:13:03.441282Z","iopub.status.idle":"2024-04-11T18:13:03.478109Z","shell.execute_reply.started":"2024-04-11T18:13:03.441252Z","shell.execute_reply":"2024-04-11T18:13:03.476444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After merge, convert back to pandas for errors='ignore' functionality \njoin_data1 = join_data1.to_pandas()\n# Based on training preprocessing\ndrop_cols = ['clientscnt_136L', 'datelastinstal40dpd_247D', 'equalitydataagreement_891L', 'equalityempfrom_62L', 'interestrategrace_34L', 'isbidproductrequest_292L', 'lastdependentsnum_448L', 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrepayingdate_696D', 'maxannuity_4075009A', 'payvacationpostpone_4187118D', 'validfrom_1069D', 'assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'dateofbirth_342D', 'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', 'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', 'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', 'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', 'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', 'pmtcount_4955617L', 'pmtcount_693L', 'riskassesment_302T', 'riskassesment_940T', 'pmts_pmtsoverdue_635A_sum', 'pmts_dpdvalue_108P_sum', 'amtdebitincoming_4809443A_sum', 'amtdebitoutgoing_4809440A_sum', 'amtdepositbalance_4809441A_sum', 'amtdepositincoming_4809444A_sum', 'amtdepositoutgoing_4809442A_sum', 'amount_1115A_sum', 'credquantity_1099L_sum', 'credquantity_984L_sum', 'debtpastduevalue_732A_sum', 'debtvalue_227A_sum', 'dpd_550P_sum', 'dpd_733P_sum', 'dpdmax_851P_max', 'installmentamount_644A_sum', 'installmentamount_833A_sum', 'instlamount_892A_sum', 'interestrateyearly_538L_max', 'maxdebtpduevalodued_3940955A_max', 'numberofinstls_810L_sum', 'overdueamountmax_950A_max', 'pmtdaysoverdue_1135P_sum', 'pmtnumpending_403L_sum', 'residualamount_3940956A_sum', 'totalamount_503A_sum', 'totalamount_881A_sum', 'amount_416A_sum']\njoin_data1 = join_data1.drop(drop_cols, axis=1, errors='ignore')\njoin_data1.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.483928Z","iopub.execute_input":"2024-04-11T18:13:03.484409Z","iopub.status.idle":"2024-04-11T18:13:03.554919Z","shell.execute_reply.started":"2024-04-11T18:13:03.484375Z","shell.execute_reply":"2024-04-11T18:13:03.553180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_static, test_static_cb, test_person_1, test_credit_bureau_b_2, test_other_1,test_credit_bureau_b_1,test_deposit_1\ndel test_person_1_feats_1, test_person_1_feats_2, test_credit_bureau_b_2_feats, test_other_1_feats, test_credit_bureau_b_1_feats, test_deposit_1_feats   \ngc.collect()  ","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.560656Z","iopub.execute_input":"2024-04-11T18:13:03.561333Z","iopub.status.idle":"2024-04-11T18:13:03.695493Z","shell.execute_reply.started":"2024-04-11T18:13:03.561297Z","shell.execute_reply":"2024-04-11T18:13:03.693856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Part 2","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 1\ntest_applprev_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_applprev_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_applprev_1_1.csv\").pipe(set_table_dtypes) \n    ], how=\"vertical_relaxed\")\n\ntest_tax_registry_a_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_a_1.csv\").pipe(set_table_dtypes)\ntest_tax_registry_b_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_b_1.csv\").pipe(set_table_dtypes)    \ntest_tax_registry_c_1 = pl.read_csv(pathway + \"csv_files/test/test_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    \ntest_credit_bureau_a_1 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_0.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_1.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_2.csv\").pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_1_3.csv\").pipe(set_table_dtypes),\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.697172Z","iopub.execute_input":"2024-04-11T18:13:03.697654Z","iopub.status.idle":"2024-04-11T18:13:03.792189Z","shell.execute_reply.started":"2024-04-11T18:13:03.697613Z","shell.execute_reply":"2024-04-11T18:13:03.790625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selection = ['credacc_actualbalance_314A', 'credacc_maxhisbal_375A', 'credacc_minhisbal_90A', 'credacc_status_367L', 'credacc_transactions_402L', 'isdebitcard_527L', 'revolvingaccount_394A']\ntest_applprev_1 = test_applprev_1.drop(selection)\n\nselection = ['annualeffectiverate_199L', 'annualeffectiverate_63L', 'contractsum_5085717L', 'credlmt_230A', 'credlmt_935A', 'debtoutstand_525A', 'debtoverdue_47A', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', 'nominalrate_281L', 'numberofcontrsvalue_258L', 'numberofcontrsvalue_358L', 'numberofinstls_320L', 'numberofoutstandinstls_59L', 'numberofoverdueinstlmaxdat_641D', 'outstandingamount_362A', 'overdueamountmax2date_1142D', 'periodicityofpmts_837L', 'prolongationcount_1120L', 'prolongationcount_599L', 'residualamount_488A', 'residualamount_856A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', 'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\ntest_credit_bureau_a_1 = test_credit_bureau_a_1.drop(selection)\n\n# Change L columns to float64\nfor col in test_credit_bureau_a_1.columns:\n        if col[-1] in (\"L\"):\n            test_credit_bureau_a_1 = test_credit_bureau_a_1.with_columns(pl.col(col).cast(pl.Float64).alias(col))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.793939Z","iopub.execute_input":"2024-04-11T18:13:03.794329Z","iopub.status.idle":"2024-04-11T18:13:03.805759Z","shell.execute_reply.started":"2024-04-11T18:13:03.794298Z","shell.execute_reply":"2024-04-11T18:13:03.804269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_1_feats_1 = test_applprev_1.group_by(\"case_id\").agg(\n    pl.col(\"actualdpd_943P\").sum().alias(\"actualdpd_943P_sum\"),\n    pl.col(\"annuity_853A\").sum().alias(\"annuity_853A_sum\"),\n    pl.col(\"byoccupationinc_3656910L\").max().alias(\"byoccupationinc_3656910L_max\"),\n    pl.col(\"childnum_21L\").max().alias(\"childnum_21L_max\"),\n    pl.col(\"credacc_credlmt_575A\").max().alias(\"credacc_credlmt_575A_max\"),\n    pl.col(\"currdebt_94A\").sum().alias(\"currdebt_94A_sum\"),\n    pl.col(\"downpmt_134A\").sum().alias(\"downpmt_134A_sum\"),\n    pl.col(\"isbidproduct_390L\").max(),\n    pl.col(\"mainoccupationinc_437A\").sum().alias(\"mainoccupationinc_437A_sum\"),\n    pl.col(\"maxdpdtolerance_577P\").max().alias(\"maxdpdtolerance_577P_max\"),\n    pl.col(\"outstandingdebt_522A\").sum().alias(\"outstandingdebt_522A_sum\"),\n    pl.col(\"pmtnum_8L\").sum().alias(\"pmtnum_8L_sum\"),\n    pl.col(\"tenor_203L\").sum().alias(\"tenor_203L_sum\"))\n\ntest_applprev_1_feats_2 = test_applprev_1.select([\"case_id\", \"num_group1\",\n    \"credtype_587L\",\"familystate_726L\",\"inittransactioncode_279L\",\"status_219L\"]).filter(\n    pl.col(\"num_group1\")==0).drop(\"num_group1\")\n\ntest_tax_registry_a_1_feats = test_tax_registry_a_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4527230A\").sum().alias(\"amount_4527230A_sum\"))\n\ntest_tax_registry_b_1_feats = test_tax_registry_b_1.group_by(\"case_id\").agg(\n    pl.col(\"amount_4917619A\").sum().alias(\"amount_4917619A_sum\"))\n\ntest_tax_registry_c_1_feats = test_tax_registry_c_1.group_by(\"case_id\").agg(\n    pl.col(\"pmtamount_36A\").sum().alias(\"pmtamount_36A_sum\"))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.808041Z","iopub.execute_input":"2024-04-11T18:13:03.808510Z","iopub.status.idle":"2024-04-11T18:13:03.833974Z","shell.execute_reply.started":"2024-04-11T18:13:03.808475Z","shell.execute_reply":"2024-04-11T18:13:03.832196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_credit_bureau_a_1_feats = test_credit_bureau_a_1.group_by(\"case_id\").agg(\n    pl.col(\"dpdmax_139P\").max().alias(\"dpdmax_139P_max\"),\n    pl.col(\"dpdmax_757P\").max().alias(\"dpdmax_757P_max\"),\n    pl.col(\"monthlyinstlamount_332A\").sum().alias(\"monthlyinstlamount_332A_sum\"),\n    pl.col(\"monthlyinstlamount_674A\").sum().alias(\"monthlyinstlamount_674A_sum\"),\n    pl.col(\"nominalrate_498L\").max().alias(\"nominalrate_498L_max\"),\n    pl.col(\"numberofinstls_229L\").sum().alias(\"numberofinstls_229L_sum\"),\n    pl.col(\"numberofoutstandinstls_520L\").sum().alias(\"numberofoutstandinstls_520L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1039L\").sum().alias(\"numberofoverdueinstlmax_1039L_sum\"),\n    pl.col(\"numberofoverdueinstlmax_1151L\").sum().alias(\"numberofoverdueinstlmax_1151L_sum\"),\n    pl.col(\"numberofoverdueinstls_725L\").sum().alias(\"numberofoverdueinstls_725L_sum\"),\n    pl.col(\"numberofoverdueinstls_834L\").sum().alias(\"numberofoverdueinstls_834L_sum\"),\n    pl.col(\"outstandingamount_354A\").sum().alias(\"outstandingamount_354A_sum\"),\n    pl.col(\"overdueamount_31A\").sum().alias(\"overdueamount_31A_sum\"),\n    pl.col(\"overdueamount_659A\").sum().alias(\"overdueamount_659A_sum\"),\n    pl.col(\"overdueamountmax2_14A\").max().alias(\"overdueamountmax2_14A_max\"),\n    pl.col(\"overdueamountmax2_398A\").max().alias(\"overdueamountmax2_398A_max\"),\n    pl.col(\"overdueamountmax_155A\").max().alias(\"overdueamountmax_155A_max\"),\n    pl.col(\"overdueamountmax_35A\").max().alias(\"overdueamountmax_35A_max\"),\n    pl.col(\"periodicityofpmts_1102L\").max().alias(\"periodicityofpmts_1102L_max\"),\n    pl.col(\"totalamount_6A\").sum().alias(\"totalamount_6A_sum\"))\n\ntest_tax_registry_c_1_feats= test_tax_registry_c_1_feats.with_columns(pl.col('case_id').cast(pl.Int64))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.835415Z","iopub.execute_input":"2024-04-11T18:13:03.835765Z","iopub.status.idle":"2024-04-11T18:13:03.849774Z","shell.execute_reply.started":"2024-04-11T18:13:03.835736Z","shell.execute_reply":"2024-04-11T18:13:03.848895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data2 = test_basetable.join(test_applprev_1_feats_1, how=\"left\", on=\"case_id\"\n).join(test_applprev_1_feats_2, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_a_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_b_1_feats, how=\"left\", on=\"case_id\"\n).join(test_tax_registry_c_1_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_1_feats, how=\"left\", on=\"case_id\")\n\njoin_data2=join_data2.to_pandas()\n\ndrop_cols = ['date_decision','MONTH','WEEK_NUM','amount_4917619A_sum']\njoin_data2 = join_data2.drop(drop_cols, axis=1, errors='ignore')\njoin_data2.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.850941Z","iopub.execute_input":"2024-04-11T18:13:03.852068Z","iopub.status.idle":"2024-04-11T18:13:03.880648Z","shell.execute_reply.started":"2024-04-11T18:13:03.852035Z","shell.execute_reply":"2024-04-11T18:13:03.879211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_1, test_tax_registry_a_1, test_tax_registry_b_1, test_tax_registry_c_1,test_credit_bureau_a_1\ndel test_applprev_1_feats_1, test_applprev_1_feats_2, test_tax_registry_a_1_feats, test_tax_registry_b_1_feats, test_tax_registry_c_1_feats, test_credit_bureau_a_1_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.882561Z","iopub.execute_input":"2024-04-11T18:13:03.883119Z","iopub.status.idle":"2024-04-11T18:13:03.997551Z","shell.execute_reply.started":"2024-04-11T18:13:03.883075Z","shell.execute_reply":"2024-04-11T18:13:03.995710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Part 3","metadata":{}},{"cell_type":"code","source":"# Additional testing data, depth = 2\ntest_applprev_2 = pl.read_csv(pathway + \"csv_files/test/test_applprev_2.csv\").pipe(set_table_dtypes)\n\ntest_person_2 = pl.read_csv(pathway + \"csv_files/test/test_person_2.csv\").pipe(set_table_dtypes)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:03.999700Z","iopub.execute_input":"2024-04-11T18:13:04.000774Z","iopub.status.idle":"2024-04-11T18:13:04.021553Z","shell.execute_reply.started":"2024-04-11T18:13:04.000732Z","shell.execute_reply":"2024-04-11T18:13:04.019938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel = ['case_id','num_group1','num_group2','pmts_dpd_1073P','pmts_dpd_303P','pmts_overdue_1140A','pmts_overdue_1152A']\ntest_credit_bureau_a_2 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_0.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_1.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_2.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_3.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_4.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_5.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")\n\ntest_credit_bureau_a_21 = pl.concat(\n    [pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_6.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_7.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_8.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_9.csv\",columns=sel).pipe(set_table_dtypes),\n    pl.read_csv(pathway + \"csv_files/test/test_credit_bureau_a_2_10.csv\",columns=sel).pipe(set_table_dtypes)\n    ], how=\"vertical_relaxed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.023470Z","iopub.execute_input":"2024-04-11T18:13:04.023931Z","iopub.status.idle":"2024-04-11T18:13:04.110963Z","shell.execute_reply.started":"2024-04-11T18:13:04.023895Z","shell.execute_reply":"2024-04-11T18:13:04.109371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_applprev_2_feats = test_applprev_2.select([\"case_id\", \"num_group1\", \"num_group2\",\n    \"conts_type_509L\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\") \n\ntest_credit_bureau_a_2_feats = test_credit_bureau_a_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"))\n\ntest_credit_bureau_a_21_feats = test_credit_bureau_a_21.group_by(\"case_id\").agg(\n    pl.col(\"pmts_dpd_1073P\").sum().alias(\"pmts_dpd_1073P_sum\"),\n    pl.col(\"pmts_dpd_303P\").sum().alias(\"pmts_dpd_303P_sum\"),\n    pl.col(\"pmts_overdue_1140A\").sum().alias(\"pmts_overdue_1140A_sum\"),\n    pl.col(\"pmts_overdue_1152A\").sum().alias(\"pmts_overdue_1152A_sum\"))\n\ntest_person_2_feats = test_person_2.select([\"case_id\", \"num_group1\", \"num_group2\", \"addres_zip_823M\",\n    \"addres_district_368M\",\"conts_role_79M\",\"empls_economicalst_849M\",\"empls_employer_name_740M\"]).filter(\n    (pl.col(\"num_group1\")==0) & (pl.col(\"num_group2\")==0)).drop(\"num_group1\").drop(\"num_group2\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.112825Z","iopub.execute_input":"2024-04-11T18:13:04.113257Z","iopub.status.idle":"2024-04-11T18:13:04.131304Z","shell.execute_reply.started":"2024-04-11T18:13:04.113223Z","shell.execute_reply":"2024-04-11T18:13:04.129857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_data3 = test_basetable.join(test_applprev_2_feats, how=\"left\", on=\"case_id\"\n).join(test_credit_bureau_a_2_feats, how=\"left\", on=\"case_id\").join(test_credit_bureau_a_21_feats, how=\"left\", on=\"case_id\").join(test_person_2_feats, how=\"left\", on=\"case_id\")\n\njoin_data3 = join_data3.to_pandas()\njoin_data3 = join_data3.drop(['date_decision','MONTH','WEEK_NUM'], axis=1, errors='ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.135410Z","iopub.execute_input":"2024-04-11T18:13:04.135896Z","iopub.status.idle":"2024-04-11T18:13:04.151723Z","shell.execute_reply.started":"2024-04-11T18:13:04.135864Z","shell.execute_reply":"2024-04-11T18:13:04.150361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_applprev_2, test_person_2, test_credit_bureau_a_2,test_credit_bureau_a_21\ndel test_applprev_2_feats, test_credit_bureau_a_2_feats, test_person_2_feats, test_credit_bureau_a_21_feats\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.152944Z","iopub.execute_input":"2024-04-11T18:13:04.153855Z","iopub.status.idle":"2024-04-11T18:13:04.321703Z","shell.execute_reply.started":"2024-04-11T18:13:04.153744Z","shell.execute_reply":"2024-04-11T18:13:04.320352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [join_data1, join_data2, join_data3]\njoin_test = reduce(lambda left, right: pd.merge(left, right, on='case_id'), dfs)\n\n# Convert back to polars for datetime \njoin_test = pl.from_pandas(join_test)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.323694Z","iopub.execute_input":"2024-04-11T18:13:04.324448Z","iopub.status.idle":"2024-04-11T18:13:04.386529Z","shell.execute_reply.started":"2024-04-11T18:13:04.324404Z","shell.execute_reply":"2024-04-11T18:13:04.385147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"join_test.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.389135Z","iopub.execute_input":"2024-04-11T18:13:04.389547Z","iopub.status.idle":"2024-04-11T18:13:04.414871Z","shell.execute_reply.started":"2024-04-11T18:13:04.389513Z","shell.execute_reply":"2024-04-11T18:13:04.413404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Join Test Data","metadata":{}},{"cell_type":"code","source":"test = join_test.with_columns(pl.col('date_decision').cast(pl.Date))\n# Feature engineer date columns\ndate_list = ['datefirstoffer_1144D', 'datelastunpaid_3546854D', 'dtlastpmtallstes_4499206D', 'firstclxcampaign_1125D', 'lastdelinqdate_224D', 'lastrejectdate_50D', 'maxdpdinstldate_3546855D', 'birthdate_574D', 'responsedate_1012D', 'responsedate_4527233D', 'responsedate_4917613D', 'empl_employedfrom_271D',\n            'firstdatedue_489D','lastactivateddate_801D','lastapplicationdate_877D', 'lastapprdate_640D', 'dateofbirth_337D','birth_259D']\n\nfor col in date_list:\n    test = test.with_columns(pl.col(col).cast(pl.Date))\n    test = test.with_columns(\n        ((pl.col(\"date_decision\") - pl.col(col)) / (24 * 60 * 60 * 1000)).cast(pl.Float64).alias(f\"{col}_diff\"))\n\ntest = test.pipe(set_table_dtypes).pipe(convert_strings)\n\ndrop_list = ['date_decision','MONTH'] + date_list\n# Convert to pandas for drop(errors='ignore')\ntest = test.to_pandas()\ntest = test.drop(drop_list, axis=1, errors='ignore')\n\ndel join_data1, join_data2, join_data3, join_test, dfs\ngc.collect()\n\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.416815Z","iopub.execute_input":"2024-04-11T18:13:04.417273Z","iopub.status.idle":"2024-04-11T18:13:04.646959Z","shell.execute_reply.started":"2024-04-11T18:13:04.417234Z","shell.execute_reply":"2024-04-11T18:13:04.645436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noticed some of these numeric variables were parsed as strings, changed them all to int \nnum_list = ['numinstlswithdpd5_4187116L', 'numinstmatpaidtearly2d_4499204L','numinstpaid_4499208L','numinstpaidearly3dest_4493216L','numinstpaidearly5dest_4493211L',\n           'numinstpaidearly5dobd_4499205L','numinstpaidearlyest_4493214L','numinstpaidlastcontr_4325080L','numinstregularpaidest_4493210L','numinsttopaygrest_4493213L','numinstunpaidmaxest_4493212L',\n           'contractssum_5085716L','days120_123L','days180_256L','days360_512L','firstquarter_103L','fourthquarter_440L','numberofqueries_373L','pmtscount_423L','secondquarter_766L','thirdquarter_1082L',\n           'days30_165L','days90_310L']\nfor col in num_list:\n    test[col]=test[col].astype('float64')","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.649023Z","iopub.execute_input":"2024-04-11T18:13:04.649439Z","iopub.status.idle":"2024-04-11T18:13:04.671209Z","shell.execute_reply.started":"2024-04-11T18:13:04.649408Z","shell.execute_reply":"2024-04-11T18:13:04.669659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same as training preprocessing\ndrop_list = ['lastapprcommoditytypec_5251766M', 'lastrejectcommodtypec_5251769M','lastrejectcommoditycat_161M','lastrejectreasonclient_4145040M',\n            'previouscontdistrict_112M','lastapprcommoditycat_1041M', 'lastcancelreason_561M','lastrejectreason_759M','commnoinclast6m_3546845L', 'deferredmnthsnum_166L', 'mastercontrelectronic_519L', 'mastercontrexist_109L',\n            'bankacctype_710L','cardtype_51L','isdebitcard_729L','lastapprcommoditytypec_5251766M','lastrejectcommodtypec_5251769M','opencred_647L','paytype1st_925L','paytype_783L',\n            'twobodfilling_608L','typesuite_864L','education_88M','maritalst_893M','type_25L','safeguarantyflag_411L','addres_zip_823M','addres_district_368M','conts_role_79M','empls_economicalst_849M','empls_employer_name_740M']\ntest.drop(columns=drop_list,axis=1,errors='ignore',inplace=True)\n# Drop rows with all na\ntest.dropna(axis=1, how='all', inplace=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.672801Z","iopub.execute_input":"2024-04-11T18:13:04.673172Z","iopub.status.idle":"2024-04-11T18:13:04.735052Z","shell.execute_reply.started":"2024-04-11T18:13:04.673142Z","shell.execute_reply":"2024-04-11T18:13:04.733812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Imputation","metadata":{}},{"cell_type":"code","source":"\nprint(np.count_nonzero(test.isnull()))\n# Impute missing values, 0 missing values after imputation\ntest = imputer(test)\nprint(np.count_nonzero(test.isnull()))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.737038Z","iopub.execute_input":"2024-04-11T18:13:04.739249Z","iopub.status.idle":"2024-04-11T18:13:04.865625Z","shell.execute_reply.started":"2024-04-11T18:13:04.739199Z","shell.execute_reply":"2024-04-11T18:13:04.864446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See boolean columns\nbool_cols = test.select_dtypes(include=['bool']).columns.tolist()\n# Convert boolean columns to 0 or 1 (False or True)\nfor col in bool_cols:\n    test[col] = test[col].astype(int)\n# Check unique values of bool_cols\nfor col in bool_cols:\n    print(test[col].unique())","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.875750Z","iopub.execute_input":"2024-04-11T18:13:04.876512Z","iopub.status.idle":"2024-04-11T18:13:04.889684Z","shell.execute_reply.started":"2024-04-11T18:13:04.876465Z","shell.execute_reply":"2024-04-11T18:13:04.888420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify dtypes are correct\n#pd.set_option('display.max_rows', None)\n#test.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.891655Z","iopub.execute_input":"2024-04-11T18:13:04.892499Z","iopub.status.idle":"2024-04-11T18:13:04.898120Z","shell.execute_reply.started":"2024-04-11T18:13:04.892457Z","shell.execute_reply":"2024-04-11T18:13:04.896683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = test.select_dtypes(include=['category', 'object']).columns.tolist()\n# Create dummies for all cat columns, not dropping first to keep column names same as training\ntest = pd.get_dummies(test, dtype=int, columns=cat_cols, sparse=True, drop_first=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.899758Z","iopub.execute_input":"2024-04-11T18:13:04.900183Z","iopub.status.idle":"2024-04-11T18:13:04.983058Z","shell.execute_reply.started":"2024-04-11T18:13:04.900151Z","shell.execute_reply":"2024-04-11T18:13:04.981863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = reduce_mem_usage(test)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:04.984907Z","iopub.execute_input":"2024-04-11T18:13:04.985630Z","iopub.status.idle":"2024-04-11T18:13:05.123537Z","shell.execute_reply.started":"2024-04-11T18:13:04.985590Z","shell.execute_reply":"2024-04-11T18:13:05.122016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Model","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv('/kaggle/input/training/train_dummy305_withimpute.csv').pipe(set_table_dtypes).pipe(convert_strings)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:05.124943Z","iopub.execute_input":"2024-04-11T18:13:05.125295Z","iopub.status.idle":"2024-04-11T18:13:18.263391Z","shell.execute_reply.started":"2024-04-11T18:13:05.125266Z","shell.execute_reply":"2024-04-11T18:13:18.262126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert polars df to pandas so pandas specific methods/attributes work later, seems more memory efficient to load as pl and convert to pd than load as pd\ntrain = train.to_pandas()\n# Get list of ids for submission file\nids = test['case_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:18.264968Z","iopub.execute_input":"2024-04-11T18:13:18.265336Z","iopub.status.idle":"2024-04-11T18:13:23.195005Z","shell.execute_reply.started":"2024-04-11T18:13:18.265306Z","shell.execute_reply":"2024-04-11T18:13:23.193488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train is already dummified so make sure all cols are numeric\nconvert_cols = train.select_dtypes(include=['category', 'object']).columns.tolist()\nfor col in convert_cols:\n    train[col]= pd.to_numeric(train[col])\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:23.196880Z","iopub.execute_input":"2024-04-11T18:13:23.197307Z","iopub.status.idle":"2024-04-11T18:13:23.207572Z","shell.execute_reply.started":"2024-04-11T18:13:23.197273Z","shell.execute_reply":"2024-04-11T18:13:23.206188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = reduce_mem_usage(train)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:23.209157Z","iopub.execute_input":"2024-04-11T18:13:23.210181Z","iopub.status.idle":"2024-04-11T18:13:28.799917Z","shell.execute_reply.started":"2024-04-11T18:13:23.210142Z","shell.execute_reply":"2024-04-11T18:13:28.798321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.set_option('display.max_rows', None)\n#train.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:28.802033Z","iopub.execute_input":"2024-04-11T18:13:28.803319Z","iopub.status.idle":"2024-04-11T18:13:28.807793Z","shell.execute_reply.started":"2024-04-11T18:13:28.803260Z","shell.execute_reply":"2024-04-11T18:13:28.806599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only select common columns to use\ncommon_columns = list(set(train.columns) & set(test.columns))\n\ntest=test[common_columns]\n\n# Subset train with only columns seen in test + target\ntrain = train[common_columns+['target']]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:28.809165Z","iopub.execute_input":"2024-04-11T18:13:28.810004Z","iopub.status.idle":"2024-04-11T18:13:31.044239Z","shell.execute_reply.started":"2024-04-11T18:13:28.809972Z","shell.execute_reply":"2024-04-11T18:13:31.043257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Fit on stratified sample\n# Note: no random seed, warning message is fine\ntrain_sample = train.groupby('target', group_keys=False).apply(lambda x: x.sample(frac=0.01)).reset_index(drop=True)\ntrain_sample.head()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:31.045638Z","iopub.execute_input":"2024-04-11T18:13:31.046317Z","iopub.status.idle":"2024-04-11T18:13:32.713756Z","shell.execute_reply.started":"2024-04-11T18:13:31.046282Z","shell.execute_reply":"2024-04-11T18:13:32.712620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train.loc[:,'target'].to_frame('target')\nX = train.drop(['target',], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:32.715461Z","iopub.execute_input":"2024-04-11T18:13:32.715867Z","iopub.status.idle":"2024-04-11T18:13:32.726665Z","shell.execute_reply.started":"2024-04-11T18:13:32.715835Z","shell.execute_reply":"2024-04-11T18:13:32.725143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check target distribution is same after all the preprocessing/sampling\nprint(round(y.target.value_counts()[1]/y.target.value_counts().sum(),4))","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:32.728457Z","iopub.execute_input":"2024-04-11T18:13:32.728858Z","iopub.status.idle":"2024-04-11T18:13:32.740310Z","shell.execute_reply.started":"2024-04-11T18:13:32.728814Z","shell.execute_reply":"2024-04-11T18:13:32.738739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:32.742771Z","iopub.execute_input":"2024-04-11T18:13:32.743216Z","iopub.status.idle":"2024-04-11T18:13:32.991399Z","shell.execute_reply.started":"2024-04-11T18:13:32.743184Z","shell.execute_reply":"2024-04-11T18:13:32.989634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Do not include case_id, or week_num as numeric \nnumeric_cols = test.select_dtypes(include=['number']).columns.tolist()\nnumeric_cols.remove('case_id')\nnumeric_cols.remove('WEEK_NUM')\n#print(numeric_cols)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:32.993211Z","iopub.execute_input":"2024-04-11T18:13:32.993614Z","iopub.status.idle":"2024-04-11T18:13:33.013045Z","shell.execute_reply.started":"2024-04-11T18:13:32.993584Z","shell.execute_reply":"2024-04-11T18:13:33.011391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\nscaler = MinMaxScaler(copy=False)\nX[numeric_cols] = scaler.fit_transform(X[numeric_cols])\ntest[numeric_cols] = scaler.transform(test[numeric_cols])\nwarnings.filterwarnings(\"default\")\n\nX.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:33.015024Z","iopub.execute_input":"2024-04-11T18:13:33.015698Z","iopub.status.idle":"2024-04-11T18:13:33.164949Z","shell.execute_reply.started":"2024-04-11T18:13:33.015661Z","shell.execute_reply":"2024-04-11T18:13:33.163249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"code","source":"# Drop case_id and week_num from features\nweeks = X[\"WEEK_NUM\"]\nX_feats = X.drop(['case_id', 'WEEK_NUM'], axis=1)\n\n# Sort columns in alphabetical order for training so columns match test submission\nX_feats = X_feats.reindex(sorted(X_feats.columns), axis=1)\n\nprint(X_feats.shape)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:33.167107Z","iopub.execute_input":"2024-04-11T18:13:33.167623Z","iopub.status.idle":"2024-04-11T18:13:33.215517Z","shell.execute_reply.started":"2024-04-11T18:13:33.167585Z","shell.execute_reply":"2024-04-11T18:13:33.213950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_feats.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:33.217412Z","iopub.execute_input":"2024-04-11T18:13:33.218025Z","iopub.status.idle":"2024-04-11T18:13:33.259260Z","shell.execute_reply.started":"2024-04-11T18:13:33.217977Z","shell.execute_reply":"2024-04-11T18:13:33.258106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:33.260610Z","iopub.execute_input":"2024-04-11T18:13:33.261421Z","iopub.status.idle":"2024-04-11T18:13:33.442723Z","shell.execute_reply.started":"2024-04-11T18:13:33.261386Z","shell.execute_reply":"2024-04-11T18:13:33.441147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwarnings.filterwarnings(\"ignore\")\ncv = StratifiedGroupKFold(n_splits=5, shuffle=True)\n\nfitted_models = []\ncv_scores = []\n\n# Note: uncomment device when running with GPU P100 accelerator\ngrid_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.03,\n    \"n_estimators\": 2000,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"random_state\": 123,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"verbose\": -1,\n    'device':'gpu',\n}\n\nfor idx_train, idx_valid in cv.split(X_feats, y, groups=weeks):\n    X_train, y_train = X_feats.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X_feats.iloc[idx_valid], y.iloc[idx_valid]\n    \n    clf = lgb.LGBMClassifier(**grid_params)\n    clf.fit(\n        X_train, y_train,\n        eval_set = [(X_valid, y_valid)],\n        callbacks = [lgb.log_evaluation(200), lgb.early_stopping(100)])\n    fitted_models.append(clf)\n    \n    y_pred_valid = clf.predict_proba(X_valid)[:,1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores.append(auc_score)\n\nprint(\"CV AUC scores: \", cv_scores)\nprint(\"Maximum CV AUC score: \", max(cv_scores))\n\nwarnings.filterwarnings(\"default\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:13:33.445078Z","iopub.execute_input":"2024-04-11T18:13:33.446152Z","iopub.status.idle":"2024-04-11T18:14:06.045722Z","shell.execute_reply.started":"2024-04-11T18:13:33.446103Z","shell.execute_reply":"2024-04-11T18:14:06.044670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the best cv model\nclass VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n\nmodel = VotingModel(fitted_models)\nmodel_plot = fitted_models[np.argmax(cv_scores)]","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.046980Z","iopub.execute_input":"2024-04-11T18:14:06.048109Z","iopub.status.idle":"2024-04-11T18:14:06.057558Z","shell.execute_reply.started":"2024-04-11T18:14:06.048074Z","shell.execute_reply":"2024-04-11T18:14:06.056241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del fitted_models, cv_scores\ngc.collect() ","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.059163Z","iopub.execute_input":"2024-04-11T18:14:06.059562Z","iopub.status.idle":"2024-04-11T18:14:06.248084Z","shell.execute_reply.started":"2024-04-11T18:14:06.059530Z","shell.execute_reply":"2024-04-11T18:14:06.245902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metric Scoring","metadata":{}},{"cell_type":"code","source":"\"\"\"\nwarnings.filterwarnings(\"ignore\")\n\nbase_train = pd.concat([X_train, y_train], axis=1)\nbase_train['score'] = grid_search.predict_proba(X_train_feats)[:,1]\nprint(f\"The AUC score on the train set is: {roc_auc_score(base_train['target'], base_train['score'])}\")\n\nbase_valid = pd.concat([X_valid, y_valid], axis=1)\nbase_valid['score'] = grid_search.predict_proba(X_valid_feats)[:,1]\nprint(f\"The AUC score on the valid set is: {roc_auc_score(base_valid['target'], base_valid['score'])}\")\n\nwarnings.filterwarnings('default')\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.250465Z","iopub.execute_input":"2024-04-11T18:14:06.250972Z","iopub.status.idle":"2024-04-11T18:14:06.260663Z","shell.execute_reply.started":"2024-04-11T18:14:06.250936Z","shell.execute_reply":"2024-04-11T18:14:06.259488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Taken from competition starter notebook made by one of the organizers \n# Note: may not work based on how we sampled a low percent of the data because some weeks will be all 0 targets\n# Using 1 and 2% samples did not work\n# Using 5% only worked for train, not for valid\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.262183Z","iopub.execute_input":"2024-04-11T18:14:06.262568Z","iopub.status.idle":"2024-04-11T18:14:06.273011Z","shell.execute_reply.started":"2024-04-11T18:14:06.262537Z","shell.execute_reply":"2024-04-11T18:14:06.271588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nwarnings.filterwarnings(\"ignore\")\n# supress lets the notebook run even if there is an exception thrown here from the sampling\nwith suppress(Exception):\n    stability_score_train = gini_stability(base_train)\n    print(f'The stability score on the train set is: {stability_score_train}') \n\nwarnings.filterwarnings(\"default\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.274524Z","iopub.execute_input":"2024-04-11T18:14:06.274946Z","iopub.status.idle":"2024-04-11T18:14:06.291956Z","shell.execute_reply.started":"2024-04-11T18:14:06.274914Z","shell.execute_reply":"2024-04-11T18:14:06.290284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nwarnings.filterwarnings(\"ignore\")\n\nwith suppress(Exception):\n    stability_score_valid = gini_stability(base_valid)\n    print(f'The stability score on the valid set is: {stability_score_valid}') \n\nwarnings.filterwarnings(\"default\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.293905Z","iopub.execute_input":"2024-04-11T18:14:06.294757Z","iopub.status.idle":"2024-04-11T18:14:06.303996Z","shell.execute_reply.started":"2024-04-11T18:14:06.294711Z","shell.execute_reply":"2024-04-11T18:14:06.302408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = X_feats.columns\nlength = len(X_feats.columns)\n\ndel X_feats, X_train, X_valid, y_train, y_valid, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.305645Z","iopub.execute_input":"2024-04-11T18:14:06.306136Z","iopub.status.idle":"2024-04-11T18:14:06.485595Z","shell.execute_reply.started":"2024-04-11T18:14:06.306096Z","shell.execute_reply":"2024-04-11T18:14:06.484090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance","metadata":{}},{"cell_type":"code","source":"# For lgb model, does not work with VotingModel Class\nlgb.plot_importance(model_plot, figsize=(20,60))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:06.488458Z","iopub.execute_input":"2024-04-11T18:14:06.488963Z","iopub.status.idle":"2024-04-11T18:14:09.965240Z","shell.execute_reply.started":"2024-04-11T18:14:06.488920Z","shell.execute_reply":"2024-04-11T18:14:09.963997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n# Get list of least important features\nimportances = model_plot.feature_importances_\nfeature_importance = pd.DataFrame({'importance':importances,'features':features}).sort_values('importance', ascending=False).reset_index(drop=True)\nfeature_importance\n\ndrop_list = []\nfor i, f in feature_importance.iterrows():\n    if f['importance']<100:\n        drop_list.append(f['features'])\nprint(f\"Number of original features: {length}\")     \nprint(f\"Number of features which are not important: {len(drop_list)}\")\nprint(drop_list)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:09.966484Z","iopub.execute_input":"2024-04-11T18:14:09.966944Z","iopub.status.idle":"2024-04-11T18:14:09.977192Z","shell.execute_reply.started":"2024-04-11T18:14:09.966896Z","shell.execute_reply":"2024-04-11T18:14:09.975774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\nto do: .","metadata":{}},{"cell_type":"code","source":"# Sort columns alphabetically to match loaded model\ntest = test.reindex(sorted(test.columns), axis=1)\n\npredictions = model.predict_proba(test.drop(['case_id', 'WEEK_NUM'], axis=1))\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:09.978529Z","iopub.execute_input":"2024-04-11T18:14:09.978962Z","iopub.status.idle":"2024-04-11T18:14:10.053508Z","shell.execute_reply.started":"2024-04-11T18:14:09.978929Z","shell.execute_reply":"2024-04-11T18:14:10.052161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:10.055299Z","iopub.execute_input":"2024-04-11T18:14:10.055728Z","iopub.status.idle":"2024-04-11T18:14:10.287568Z","shell.execute_reply.started":"2024-04-11T18:14:10.055693Z","shell.execute_reply":"2024-04-11T18:14:10.286374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'case_id': ids, 'score': predictions[:,1]}).set_index('case_id')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:10.289003Z","iopub.execute_input":"2024-04-11T18:14:10.289382Z","iopub.status.idle":"2024-04-11T18:14:10.308386Z","shell.execute_reply.started":"2024-04-11T18:14:10.289350Z","shell.execute_reply":"2024-04-11T18:14:10.307026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T18:14:10.310045Z","iopub.execute_input":"2024-04-11T18:14:10.310467Z","iopub.status.idle":"2024-04-11T18:14:10.324213Z","shell.execute_reply.started":"2024-04-11T18:14:10.310432Z","shell.execute_reply":"2024-04-11T18:14:10.322519Z"},"trusted":true},"execution_count":null,"outputs":[]}]}