{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.7"},"papermill":{"default_parameters":{},"duration":1221.035913,"end_time":"2024-03-11T20:40:31.474791","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-03-11T20:20:10.438878","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.calibration import calibration_curve\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import roc_auc_score\n\nimport lightgbm as lgb\nimport joblib\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2024-03-11T20:20:13.177936Z","iopub.status.busy":"2024-03-11T20:20:13.177674Z","iopub.status.idle":"2024-03-11T20:20:19.340830Z","shell.execute_reply":"2024-03-11T20:20:19.340062Z"},"papermill":{"duration":6.175783,"end_time":"2024-03-11T20:20:19.343026","exception":false,"start_time":"2024-03-11T20:20:13.167243","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pre-Fitted Voting Model","metadata":{"papermill":{"duration":0.008777,"end_time":"2024-03-11T20:20:19.361579","exception":false,"start_time":"2024-03-11T20:20:19.352802","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, ClassifierMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.381759Z","iopub.status.busy":"2024-03-11T20:20:19.381081Z","iopub.status.idle":"2024-03-11T20:20:19.387550Z","shell.execute_reply":"2024-03-11T20:20:19.386761Z"},"papermill":{"duration":0.018278,"end_time":"2024-03-11T20:20:19.389382","exception":false,"start_time":"2024-03-11T20:20:19.371104","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pipeline","metadata":{"papermill":{"duration":0.008615,"end_time":"2024-03-11T20:20:19.406942","exception":false,"start_time":"2024-03-11T20:20:19.398327","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int32))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        # chosen manually through data exploration / analysis \n        exclude_cols = [\n            \"addres_district_368M\", \"addres_role_871L\", \"addres_zip_823M\", \"amount_416A\",\n            \"amount_4527230A\", \"amount_4917619A\", \"amtdepositincoming_4809444A\", \"amtdepositoutgoing_4809442A\",\n            \"annualeffectiverate_199L\", \"applicationscnt_464L\", \"applicationscnt_629L\", \"applicationscnt_867L\",\n            \"assignmentdate_238D\", \"assignmentdate_4527235D\", \"assignmentdate_4955616D\", \"avgdpdtolclosure24_3658938P\",\n            \"avginstallast24m_3658937A\", \"bankacctype_710L\", \"birth_259D\", \"birthdate_574D\", \"birthdate_87D\",\n            \"byoccupationinc_3656910L\", \"cacccardblochreas_147M\", \"cancelreason_3545846M\", \"cardtype_51L\",\n            \"childnum_21L\", \"classificationofcontr_1114M\", \"classificationofcontr_13M\", \"classificationofcontr_400M\",\n            \"clientscnt_100L\", \"clientscnt_1022L\", \"clientscnt_1071L\", \"clientscnt_1130L\", \"clientscnt_136L\",\n            \"clientscnt_157L\", \"clientscnt_257L\", \"clientscnt_304L\", \"clientscnt_360L\", \"clientscnt_493L\",\n            \"clientscnt_533L\", \"clientscnt_887L\", \"clientscnt_946L\", \"clientscnt12m_3712952L\", \"clientscnt3m_3712950L\",\n            \"clientscnt6m_3712949L\", \"cntpmts24_3658933L\", \"contaddr_district_15M\", \"contaddr_smempladdr_334L\",\n            \"contaddr_zipcode_807M\", \"contractst_545M\", \"credamount_590A\", \"credlmt_935A\", \"credor_3940957M\",\n            \"credtype_322L\", \"credtype_587L\", \"dateofbirth_337D\", \"dateofbirth_342D\", \"dateofcredend_353D\",\n            \"days120_123L\", \"days180_256L\", \"days30_165L\", \"days90_310L\", \"description_351M\", \"description_5085714M\",\n            \"eir_270L\", \"empladdr_district_926M\", \"empladdr_zipcode_114M\", \"employername_160M\", \"empls_employedfrom_796D\",\n            \"empls_employer_name_740M\", \"equalitydataagreement_891L\", \"equalityempfrom_62L\", \"familystate_726L\",\n            \"financialinstitution_382M\", \"financialinstitution_591M\", \"firstclxcampaign_1125D\", \"firstdatedue_489D\",\n            \"firstnonzeroinstldate_307D\", \"for3years_584L\", \"formonth_118L\", \"formonth_206L\", \"formonth_535L\",\n            \"forquarter_1017L\", \"forquarter_462L\", \"forquarter_634L\", \"fortoday_1092L\", \"forweek_1077L\",\n            \"forweek_528L\", \"foryear_618L\", \"foryear_818L\", \"gender_992L\", \"housingtype_772L\", \"incometype_1044T\",\n            \"isbidproductrequest_292L\", \"isdebitcard_729L\", \"language1_981M\", \"last180dayaveragebalance_704A\",\n            \"last180dayturnover_1134A\", \"last30dayturnover_651A\", \"lastupdate_1112D\", \"lastupdate_260D\", \"lastupdate_388D\",\n            \"mainoccupationinc_384A\", \"mainoccupationinc_437A\", \"maritalst_385M\", \"maritalst_703L\", \"maritalst_893M\",\n            \"maxannuity_159A\", \"maxannuity_4075009A\", \"maxdbddpdlast1m_3658939P\", \"maxdbddpdtollast12m_3658940P\",\n            \"maxdbddpdtollast6m_4187119P\", \"maxdebt4_972A\", \"maxdebtpduevalodued_3940955A\", \"maxdpdfrom6mto36m_3546853P\",\n            \"maxdpdinstlnum_3546846P\", \"maxinstallast24m_3658928A\", \"maxoutstandbalancel12m_4187113A\", \"maxpmtlast3m_4525190A\",\n            \"mindbddpdlast24m_3658935P\", \"mindbdtollast24m_4525191P\", \"mobilephncnt_593L\", \"name_4527232M\",\n            \"name_4917606M\", \"numactivecredschannel_414L\", \"numberofinstls_810L\", \"numberofoverdueinstlmax_1039L\",\n            \"numinstpaidearly5dobd_4499205L\", \"numinstregularpaidest_4493210L\", \"numinsttopaygrest_4493213L\",\n            \"numinstunpaidmaxest_4493212L\", \"overdueamountmax_950A\", \"overdueamountmax2_14A\", \"paytype_783L\",\n            \"periodicityofpmts_997M\", \"persontype_1072L\", \"persontype_792L\", \"previouscontdistrict_112M\", \"pmtaverage_4527227A\",\n            \"pmtaverage_4955615A\", \"pmtcount_4955617L\", \"pmtcount_693L\", \"pmts_month_158T\", \"pmts_overdue_1152A\",\n            \"posfpd10lastmonth_333P\", \"posfpd30lastmonth_3976960P\", \"purposeofcred_722M\", \"refreshdate_3813885D\",\n            \"registaddr_district_1083M\", \"registaddr_zipcode_184M\", \"rejectreason_755M\", \"rejectreasonclient_4145042M\",\n            \"relatedpersons_role_762T\", \"relationshiptoclient_642T\", \"residualamount_856A\", \"responsedate_1012D\",\n            \"responsedate_4527233D\", \"responsedate_4917613D\", \"role_1084L\", \"role_993L\", \"safeguarantyflag_411L\",\n            \"score_940\", \"status_219L\", \"subjectrole_43M\", \"subjectrole_93M\", \"thirdquarter_1082L\", \"totalamount_6A\", \"subjectrole_326M\",\n            \"totalamount_881A\", \"totalsettled_863A\", \"twobodfilling_608L\", \"type_25L\", \"typesuite_864L\", \"validfrom_1069D\", \"sumoutstandtotalest_4493215A\", \"subjectroles_name_838M\",\n        ]\n\n        null_threshold = 0.5\n\n        # Exclude specified columns\n        df = df.drop(columns=exclude_cols, errors='ignore')\n\n        # Iterate over columns\n        for col in df.columns:\n            # Check if all values in the column are null\n            if df[col].isnull().all():\n                df.drop(columns=col, inplace=True)\n                continue\n            \n            # Check if column is categorical or continuous\n            if df[col].dtype == 'object' or df[col].dtype.name == 'category':  # Categorical column\n                mode_value = df[col].mode().iloc[0]  # Get the first mode value\n\n                if null_percentage > null_threshold:\n                    df.drop(columns=col, inplace=True)\n                else:\n                    df[col].fillna(mode_value, inplace=True)  # Fill null values with mode value\n                    \n            else:  # Continuous column\n                median_value = df[col].median()  # Calculate median\n                null_percentage = df[col].isnull().mean()\n\n                # If too many null values, drop the column\n                if null_percentage > null_threshold:\n                    df.drop(columns=col, inplace=True)\n                else:\n                    # Fill null values with median value\n                    df[col].fillna(median_value, inplace=True)\n\n        return df\n        ","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.425745Z","iopub.status.busy":"2024-03-11T20:20:19.425481Z","iopub.status.idle":"2024-03-11T20:20:19.437672Z","shell.execute_reply":"2024-03-11T20:20:19.436838Z"},"papermill":{"duration":0.023479,"end_time":"2024-03-11T20:20:19.439407","exception":false,"start_time":"2024-03-11T20:20:19.415928","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Automatic Aggregation","metadata":{"papermill":{"duration":0.008664,"end_time":"2024-03-11T20:20:19.457100","exception":false,"start_time":"2024-03-11T20:20:19.448436","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n    \n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.476571Z","iopub.status.busy":"2024-03-11T20:20:19.476292Z","iopub.status.idle":"2024-03-11T20:20:19.486434Z","shell.execute_reply":"2024-03-11T20:20:19.485637Z"},"papermill":{"duration":0.021569,"end_time":"2024-03-11T20:20:19.488325","exception":false,"start_time":"2024-03-11T20:20:19.466756","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### File I/O","metadata":{"papermill":{"duration":0.009719,"end_time":"2024-03-11T20:20:19.506836","exception":false,"start_time":"2024-03-11T20:20:19.497117","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        \n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n        chunks.append(df)\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df\n\ndef preprocess_dataframes(dataframes):\n    # Get common columns among all DataFrames\n    common_columns = set(dataframes[0].columns)\n    for df in dataframes[1:]:\n        common_columns.intersection_update(df.columns)\n    \n    # Select only common columns in all DataFrames\n    preprocessed_dfs = [df.select(common_columns) for df in dataframes]\n    \n    return preprocessed_dfs\n\ndef get_null_counts(data_store):\n    null_counts = {}\n    for key, value in data_store.items():\n        if key == \"df_base\":\n            df = value\n        else:\n            # Preprocess DataFrames in the value list\n            value = preprocess_dataframes(value)\n            df = pl.concat(value)\n        null_counts[key] = df.null_count()\n    return null_counts\n","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.525391Z","iopub.status.busy":"2024-03-11T20:20:19.525145Z","iopub.status.idle":"2024-03-11T20:20:19.531829Z","shell.execute_reply":"2024-03-11T20:20:19.531102Z"},"papermill":{"duration":0.018108,"end_time":"2024-03-11T20:20:19.533703","exception":false,"start_time":"2024-03-11T20:20:19.515595","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{"papermill":{"duration":0.00854,"end_time":"2024-03-11T20:20:19.550995","exception":false,"start_time":"2024-03-11T20:20:19.542455","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    # Add date-related features\n    df_base = (\n        df_base\n        .with_columns(\n            pl.col(\"date_decision\").dt.month().alias(\"month_decision\"),\n            pl.col(\"date_decision\").dt.weekday().alias(\"weekday_decision\"),\n        )\n    )\n    \n    # Join additional dataframes\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    \n    # Feature engineering for amount_1115A and maininc_215A if they exist\n    if \"amount_1115A\" in df_base.columns and \"maininc_215A\" in df_base.columns:\n        df_base = df_base.with_columns(\n            # Normalize the amounts\n            (pl.col(\"amount_1115A\") / pl.col(\"amount_1115A\").max()).alias(\"amount_1115A_norm\"),\n            (pl.col(\"maininc_215A\") / pl.col(\"maininc_215A\").max()).alias(\"maininc_215A_norm\"),\n            \n            # Log transformation (adding a small constant to avoid log(0))\n            pl.col(\"amount_1115A\").apply(lambda x: np.log(x + 1)).alias(\"amount_1115A_log\"),\n            pl.col(\"maininc_215A\").apply(lambda x: np.log(x + 1)).alias(\"maininc_215A_log\"),\n            \n            # Interaction terms\n            (pl.col(\"amount_1115A\") * pl.col(\"maininc_215A\")).alias(\"amount_maininc_interaction\"),\n            \n            # Ratios\n            (pl.col(\"amount_1115A\") / (pl.col(\"maininc_215A\") + 1)).alias(\"amount_to_maininc_ratio\")\n        )\n\n     # Feature engineering for amtdebitincoming_4809443A and amtdebitoutgoing_4809440A if they exist\n    if \"amtdebitincoming_4809443A\" in df_base.columns and \"amtdebitoutgoing_4809440A\" in df_base.columns:\n        df_base = df_base.with_columns(\n            # Calculate net debit transactions\n            (pl.col(\"amtdebitincoming_4809443A\") - pl.col(\"amtdebitoutgoing_4809440A\")).alias(\"net_debit_amount\"),\n            \n            # Inverse net debit amount (lower net implies more likely loan default)\n            (-pl.col(\"net_debit_amount\")).alias(\"inverse_net_debit_amount\"),\n            \n            # Flag for potential loan default (lower net means more likely to default)\n            (pl.col(\"net_debit_amount\") < 0).alias(\"potential_loan_default\")\n        )\n\n    # Feature engineering for payment timeliness index\n    if all(col in df_base.columns for col in [\"avgdbddpdlast24m_3658932P\", \"avgdbddpdlast3m_4187120P\", \"avgdbdtollast24m_4525197P\"]):\n        df_base = df_base.with_columns(\n            # Calculate payment timeliness index\n            (\n                pl.col(\"avgdbddpdlast24m_3658932P\") * 0.6 +\n                pl.col(\"avgdbddpdlast3m_4187120P\") * 0.3 +\n                pl.col(\"avgdbdtollast24m_4525197P\") * 0.1\n            ).alias(\"payment_timeliness_index\")\n        )\n\n    # Feature engineering for outstanding debt comparison\n    if all(col in df_base.columns for col in [\"totaloutstanddebtvalue_39A\", \"totaloutstanddebtvalue_668A\"]):\n        df_base = df_base.with_columns(\n            # Calculate difference between total outstanding debt of active and closed contracts\n            (pl.col(\"totaloutstanddebtvalue_39A\") - pl.col(\"totaloutstanddebtvalue_668A\")).alias(\"outstanding_debt_difference\"),\n            \n            # Ratio of total outstanding debt of active to closed contracts\n            (pl.col(\"totaloutstanddebtvalue_39A\") / (pl.col(\"totaloutstanddebtvalue_668A\") + 1)).alias(\"active_to_closed_debt_ratio\")\n        )\n\n     # Feature engineering for past due debt comparison\n    if all(col in df_base.columns for col in [\"totaldebtoverduevalue_178A\", \"totaldebtoverduevalue_718A\"]):\n        df_base = df_base.with_columns(\n            # Calculate difference between total past due debt of active and closed contracts\n            (pl.col(\"totaldebtoverduevalue_178A\") - pl.col(\"totaldebtoverduevalue_718A\")).alias(\"past_due_debt_difference\"),\n            \n            # Ratio of total past due debt of active to closed contracts\n            (pl.col(\"totaldebtoverduevalue_178A\") / (pl.col(\"totaldebtoverduevalue_718A\") + 1)).alias(\"active_to_closed_past_due_debt_ratio\")\n        )\n    \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    \n    return df_base","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.570143Z","iopub.status.busy":"2024-03-11T20:20:19.569547Z","iopub.status.idle":"2024-03-11T20:20:19.574976Z","shell.execute_reply":"2024-03-11T20:20:19.574273Z"},"papermill":{"duration":0.01688,"end_time":"2024-03-11T20:20:19.576796","exception":false,"start_time":"2024-03-11T20:20:19.559916","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.595389Z","iopub.status.busy":"2024-03-11T20:20:19.595137Z","iopub.status.idle":"2024-03-11T20:20:19.599640Z","shell.execute_reply":"2024-03-11T20:20:19.598867Z"},"papermill":{"duration":0.015752,"end_time":"2024-03-11T20:20:19.601370","exception":false,"start_time":"2024-03-11T20:20:19.585618","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{"papermill":{"duration":0.00861,"end_time":"2024-03-11T20:20:19.619092","exception":false,"start_time":"2024-03-11T20:20:19.610482","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"\n","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.637746Z","iopub.status.busy":"2024-03-11T20:20:19.637467Z","iopub.status.idle":"2024-03-11T20:20:19.641419Z","shell.execute_reply":"2024-03-11T20:20:19.640728Z"},"papermill":{"duration":0.015255,"end_time":"2024-03-11T20:20:19.643247","exception":false,"start_time":"2024-03-11T20:20:19.627992","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Files Read & Feature Engineering","metadata":{"papermill":{"duration":0.008715,"end_time":"2024-03-11T20:20:19.660605","exception":false,"start_time":"2024-03-11T20:20:19.651890","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:20:19.678920Z","iopub.status.busy":"2024-03-11T20:20:19.678686Z","iopub.status.idle":"2024-03-11T20:22:26.542632Z","shell.execute_reply":"2024-03-11T20:22:26.541531Z"},"papermill":{"duration":126.876005,"end_time":"2024-03-11T20:22:26.545249","exception":false,"start_time":"2024-03-11T20:20:19.669244","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"train data:\\t\", df_train)\n","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:22:26.565337Z","iopub.status.busy":"2024-03-11T20:22:26.564999Z","iopub.status.idle":"2024-03-11T20:22:38.552398Z","shell.execute_reply":"2024-03-11T20:22:38.551267Z"},"papermill":{"duration":11.999532,"end_time":"2024-03-11T20:22:38.554478","exception":false,"start_time":"2024-03-11T20:22:26.554946","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Files Read & Feature Engineering","metadata":{"papermill":{"duration":0.009104,"end_time":"2024-03-11T20:22:38.572989","exception":false,"start_time":"2024-03-11T20:22:38.563885","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:22:38.592481Z","iopub.status.busy":"2024-03-11T20:22:38.591784Z","iopub.status.idle":"2024-03-11T20:22:39.168839Z","shell.execute_reply":"2024-03-11T20:22:39.167993Z"},"papermill":{"duration":0.589237,"end_time":"2024-03-11T20:22:39.171145","exception":false,"start_time":"2024-03-11T20:22:38.581908","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\n\nprint(\"test data shape:\\t\", df_test.shape)\nprint(\"test data:\\t\", df_test)\n","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:22:39.190830Z","iopub.status.busy":"2024-03-11T20:22:39.190540Z","iopub.status.idle":"2024-03-11T20:22:39.230327Z","shell.execute_reply":"2024-03-11T20:22:39.229525Z"},"papermill":{"duration":0.051709,"end_time":"2024-03-11T20:22:39.232155","exception":false,"start_time":"2024-03-11T20:22:39.180446","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Elimination","metadata":{"papermill":{"duration":0.008776,"end_time":"2024-03-11T20:22:39.250093","exception":false,"start_time":"2024-03-11T20:22:39.241317","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Convert train data to pandas and filter\ndf_train, cat_cols = to_pandas(df_train)\ndf_train_filtered = df_train.pipe(Pipeline.filter_cols)\n\nprint(\"Train data shape:\\t\", df_train_filtered.shape)\nprint(\"Train data:\\n\", df_train_filtered)\n\n\n# Convert test data to pandas and filter\ndf_test, cat_cols = to_pandas(df_test, cat_cols)\ndf_test_filtered = df_test.pipe(Pipeline.filter_cols)\n\nprint(\"Test data shape:\\t\", df_test_filtered.shape)\nprint(\"Test data:\\n\", df_test_filtered)\n \n# Ensure that both train and test data have the same columns except for the target attribute\ncommon_columns = list(set(df_train_filtered.columns).intersection(set(df_test_filtered.columns)))\n\n# Add specific columns to the common columns list if they are not already present\ncolumns_to_keep = ['case_id', 'WEEK_NUM', 'month_decision', 'weekday_decision']\nfor col in columns_to_keep:\n    if col not in common_columns:\n        common_columns.append(col)\n\n# Preserve the original order of columns in df_train_filtered\ncommon_columns = [col for col in df_train_filtered.columns if col in common_columns]\n\n# Add 'target' column to train data\ndf_train_filtered = df_train_filtered[common_columns + ['target']]\n\n# Filter test data to include only common columns\ndf_test_filtered = df_test_filtered[common_columns]\n\nprint(\"Filtered train data shape:\\t\", df_train_filtered.shape)\nprint(\"Filtered train data:\\n\", df_train_filtered)\n\nprint(\"Filtered test data shape:\\t\", df_test_filtered.shape)\nprint(\"Filtered test data:\\n\", df_test_filtered)\n\ntarget_counts = df_train_filtered['target'].value_counts()\nprint(target_counts)\n\n# check to see if there are any duplicates in the data\nprint(\"Train is duplicated:\\t\", df_train_filtered[\"case_id\"].duplicated().any())\nprint(\"Train Week Range:\\t\", (df_train_filtered[\"WEEK_NUM\"].min(), df_train_filtered[\"WEEK_NUM\"].max()))\n\nprint()\n\n# check to see if there are any duplicates in the data\nprint(\"Test is duplicated:\\t\", df_test_filtered[\"case_id\"].duplicated().any())\nprint(\"Test Week Range:\\t\", (df_test_filtered[\"WEEK_NUM\"].min(), df_test_filtered[\"WEEK_NUM\"].max()))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Garbage Collection","metadata":{"papermill":{"duration":0.00925,"end_time":"2024-03-11T20:23:00.767926","exception":false,"start_time":"2024-03-11T20:23:00.758676","status":"completed"},"tags":[]}},{"cell_type":"code","source":"del data_store\n\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:23:00.787299Z","iopub.status.busy":"2024-03-11T20:23:00.786797Z","iopub.status.idle":"2024-03-11T20:23:00.911801Z","shell.execute_reply":"2024-03-11T20:23:00.911010Z"},"papermill":{"duration":0.137016,"end_time":"2024-03-11T20:23:00.913914","exception":false,"start_time":"2024-03-11T20:23:00.776898","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{"papermill":{"duration":0.009084,"end_time":"2024-03-11T20:23:00.932380","exception":false,"start_time":"2024-03-11T20:23:00.923296","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# print df train and test\nprint(\"Train data shape:\\t\", df_train_filtered.shape)\nprint(\"Train data:\\t\", df_train_filtered)\n\nprint(\"Test data shape:\\t\", df_test_filtered.shape)\nprint(\"Test data:\\t\", df_test_filtered)\n\n# Correlation matrix\n# Select only numeric columns\nnumeric_cols = df_train_filtered.select_dtypes(include=['int64', 'float64']).columns\ndf_numeric = df_train_filtered[numeric_cols]\n\n# Calculate the correlation matrix\ncorrelation_matrix = df_numeric.corr()\n\nplt.figure(figsize=(32, 24))\nsns.heatmap(correlation_matrix, annot=True, fmt=\".2f\", cmap=\"coolwarm\")\nplt.title('Correlation Matrix')\nplt.show()\n\n# Flatten the correlation matrix\ncorrelation_pairs = correlation_matrix.unstack()\n\n# Remove self-correlations\ncorrelation_pairs = correlation_pairs[correlation_pairs.index.get_level_values(0) != correlation_pairs.index.get_level_values(1)]\n\n# Sort by absolute value\nsorted_correlation_pairs = correlation_pairs.reindex(correlation_pairs.abs().sort_values(ascending=False).index)\n\n# Print the most correlated features where its above 0.95\nprint(\"Most correlated features:\")\nprint(sorted_correlation_pairs[sorted_correlation_pairs.abs() > 0.95])\n","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:23:00.952356Z","iopub.status.busy":"2024-03-11T20:23:00.951810Z","iopub.status.idle":"2024-03-11T20:23:00.978038Z","shell.execute_reply":"2024-03-11T20:23:00.977167Z"},"papermill":{"duration":0.038139,"end_time":"2024-03-11T20:23:00.979829","exception":false,"start_time":"2024-03-11T20:23:00.941690","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove highly correlated features (through this, realised got some data that exists outside the given feature definition (max_persontype) ; chosen to drop are mainly through feature definition revision, some are removed because of them having too many correlations with alot of features at once (also by comparing which one is relevant))\ndf_train_filtered = df_train_filtered.drop(columns=[\"numberofqueries_373L\", \"max_persontype_792L\", \"max_persontype_1072L\", \n                                                    \"currdebt_22A\", \"numinstlallpaidearly3d_817L\", \"sumoutstandtotal_3546847A\", \"numinstpaidearly_338L\",\n                                                    \"pctinstlsallpaidlate6d_3546844L\", \"numinsttopaygr_769L\", \"numinstlsallpaid_934L\"])\n\ndf_test_filtered = df_test_filtered.drop(columns=[\"numberofqueries_373L\", \"max_persontype_792L\", \"max_persontype_1072L\", \n                                                    \"currdebt_22A\", \"numinstlallpaidearly3d_817L\", \"sumoutstandtotal_3546847A\", \"numinstpaidearly_338L\",\n                                                    \"pctinstlsallpaidlate6d_3546844L\", \"numinsttopaygr_769L\", \"numinstlsallpaid_934L\"])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count plots for categorical features\ncategorical_cols = df_train_filtered.select_dtypes(include=['object', 'category']).columns\n\nfor col in categorical_cols:\n    plt.figure(figsize=(10, 6))\n    sns.countplot(y=col, data=df_train_filtered, order=df_train_filtered[col].value_counts().index)\n    plt.title(f'Distribution of {col}')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature engineering to combine columns\ndf_train_filtered['max_role_combined'] = df_train_filtered['max_role_993L'].astype(str) + '_' + df_train_filtered['max_role_1084L'].astype(str)\n\ndf_test_filtered['max_role_combined'] = df_test_filtered['max_role_993L'].astype(str) + '_' + df_test_filtered['max_role_1084L'].astype(str)\n\n# Drop the original columns\ndf_train_filtered.drop(columns=['max_role_993L', 'max_role_1084L'], inplace=True)\ndf_test_filtered.drop(columns=['max_role_993L', 'max_role_1084L'], inplace=True)\n\n# Label encode the combined column\nlabel_encoder = LabelEncoder()\ndf_train_filtered['max_role_combined'] = label_encoder.fit_transform(df_train_filtered['max_role_combined'])\ndf_test_filtered['max_role_combined'] = label_encoder.transform(df_test_filtered['max_role_combined'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training","metadata":{"papermill":{"duration":0.01291,"end_time":"2024-03-11T20:23:17.567408","exception":false,"start_time":"2024-03-11T20:23:17.554498","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X = df_train_filtered.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"]) # drop because it is not a feature for predicting \ny = df_train_filtered[\"target\"]\nweeks = df_train_filtered[\"WEEK_NUM\"]\n\n# model training with cross validation\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    # \"device\": \"gpu\",\n    \"max_bin\": 128\n}\n\nfitted_models = []\n\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    # 8:2 split (train:valid)\n    X_train, y_train = X.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = X.iloc[idx_valid], y.iloc[idx_valid]\n\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(100), lgb.early_stopping(100)]\n    )\n\n    fitted_models.append(model)\n\n# use voting model to determine best model \nmodel = VotingModel(fitted_models)","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:23:17.595206Z","iopub.status.busy":"2024-03-11T20:23:17.594787Z","iopub.status.idle":"2024-03-11T20:40:29.888472Z","shell.execute_reply":"2024-03-11T20:40:29.887312Z"},"papermill":{"duration":1032.311042,"end_time":"2024-03-11T20:40:29.890585","exception":false,"start_time":"2024-03-11T20:23:17.579543","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction","metadata":{"papermill":{"duration":0.015335,"end_time":"2024-03-11T20:40:29.921663","exception":false,"start_time":"2024-03-11T20:40:29.906328","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X_test = df_test_filtered.drop(columns=[\"WEEK_NUM\"])\nX_test = X_test.set_index(\"case_id\")\ny_pred_probs = model.predict_proba(X_test)[:, 1]\n\ny_pred = pd.Series(y_pred_probs, index=X_test.index)\n\n# save the trained and chosen model\njoblib.dump(model, \"trained_model.pkl\")\n","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:40:29.953970Z","iopub.status.busy":"2024-03-11T20:40:29.953671Z","iopub.status.idle":"2024-03-11T20:40:30.231819Z","shell.execute_reply":"2024-03-11T20:40:30.230927Z"},"papermill":{"duration":0.29704,"end_time":"2024-03-11T20:40:30.234187","exception":false,"start_time":"2024-03-11T20:40:29.937147","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize lists to store the true probabilities and predicted probabilities\ntrue_probs = []\npred_probs = []\n\n# Iterate over each fold to collect the predicted probabilities\nfor idx_train, idx_valid in cv.split(X, y, groups=weeks):\n    X_valid_fold, y_valid_fold = X.iloc[idx_valid], y.iloc[idx_valid]\n    pred_probs_fold = model.predict_proba(X_valid_fold)[:, 1]\n    true_probs_fold = y_valid_fold\n    true_probs.extend(true_probs_fold)\n    pred_probs.extend(pred_probs_fold)\n\n# Calculate calibration curve using all predictions\nprob_true, prob_pred = calibration_curve(true_probs, pred_probs, n_bins=15)\n\n# Plot calibration curve\nplt.plot(prob_pred, prob_true, marker='o', linestyle='-', label='Calibration curve')\nplt.plot([0, 1], [0, 1], linestyle='--', label='Perfectly calibrated')\nplt.xlabel('Mean predicted probability')\nplt.ylabel('Fraction of positives')\nplt.legend()\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Step 1: Calculate Gini scores for each WEEK_NUM\n# Initialize an empty list to store Gini scores for each WEEK_NUM\ngini_scores = []\n\n# Iterate over unique WEEK_NUM values\nfor week_num in df_train_filtered[\"WEEK_NUM\"].unique():\n    # Filter data for the current week\n    week_data = df_train_filtered[df_train_filtered[\"WEEK_NUM\"] == week_num]\n    \n    # Extract features and target for the current week\n    X_week = week_data.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])\n    y_week = week_data[\"target\"]\n    \n    # Predict probabilities for the current week\n    pred_probs_week = model.predict_proba(X_week)[:, 1]\n    \n    # Calculate AUC-ROC for the current week\n    auc_week = roc_auc_score(y_week, pred_probs_week)\n    \n    # Calculate Gini score for the current week\n    gini_week = 2 * auc_week - 1\n    \n    # Append the Gini score to the list\n    gini_scores.append(gini_week)\n\n# Ensure the number of samples in WEEK_NUM matches the number of calculated Gini scores\nweeks_unique = df_train_filtered[\"WEEK_NUM\"].unique()\nif len(weeks_unique) != len(gini_scores):\n    raise ValueError(\"Inconsistent number of samples between WEEK_NUM and Gini scores\")\n\n# Step 2: Fit a linear regression through the weekly Gini scores\nregression_model = LinearRegression()\nregression_model.fit(np.array(weeks_unique).reshape(-1, 1), gini_scores)\na = regression_model.coef_[0]\nb = regression_model.intercept_\n\n# Step 3: Calculate the falling rate (falling_rate)\nfalling_rate = min(0, a)\n\n# Step 4: Calculate the standard deviation of the residuals from the linear regression\nresiduals = gini_scores - (a * np.array(weeks_unique) + b)\nstd_residuals = np.std(residuals)\n\n# Step 5: Apply penalties to model variability\npenalty = 0.5 * std_residuals\n\n# Step 6: Calculate the final stability metric\nstability_metric = np.mean(gini_scores) + 88.0 * falling_rate - penalty\n\nprint(\"Stability Metric: \", str(stability_metric))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{"papermill":{"duration":0.015506,"end_time":"2024-03-11T20:40:30.266313","exception":false,"start_time":"2024-03-11T20:40:30.250807","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = y_pred","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:40:30.299204Z","iopub.status.busy":"2024-03-11T20:40:30.298476Z","iopub.status.idle":"2024-03-11T20:40:30.315227Z","shell.execute_reply":"2024-03-11T20:40:30.314249Z"},"papermill":{"duration":0.035461,"end_time":"2024-03-11T20:40:30.317145","exception":false,"start_time":"2024-03-11T20:40:30.281684","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Check null: \", df_subm[\"score\"].isnull().any())\n\ndf_subm_sorted = df_subm.sort_values(by=\"score\", ascending=False)\ntop_5_scores = df_subm_sorted.head()\nprint(top_5_scores)","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:40:30.349432Z","iopub.status.busy":"2024-03-11T20:40:30.349153Z","iopub.status.idle":"2024-03-11T20:40:30.362280Z","shell.execute_reply":"2024-03-11T20:40:30.361363Z"},"papermill":{"duration":0.031645,"end_time":"2024-03-11T20:40:30.364269","exception":false,"start_time":"2024-03-11T20:40:30.332624","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subm_sorted.to_csv(\"/kaggle/working/submission.csv\")","metadata":{"execution":{"iopub.execute_input":"2024-03-11T20:40:30.397489Z","iopub.status.busy":"2024-03-11T20:40:30.397241Z","iopub.status.idle":"2024-03-11T20:40:30.403177Z","shell.execute_reply":"2024-03-11T20:40:30.402519Z"},"papermill":{"duration":0.024175,"end_time":"2024-03-11T20:40:30.404950","exception":false,"start_time":"2024-03-11T20:40:30.380775","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}