{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":166996856,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Linear Regression with VIF predictor reduction and outlier reduction\n\nNew notebook starting with the base at https://www.kaggle.com/code/rickpack/english-comments-eda-linearregression.\nThe original version of that notebook lacked VIF predictor reduction and outlier reduction.\n\nWhile all assumptions (any?) of linear regression are not met by the data, this is an exploratory notebook.\n\nI am also using streamlined data loading and feature engineering to see if that causes a change from the score of 0.5 from forked notebook https://www.kaggle.com/code/yunsuxiaozi/home-credit-linearregression-is-all-you-need.\n\nData loading and feature engineering borrowed from https://www.kaggle.com/code/greysky/home-credit-baseline.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Tweak sample_run parameter when ready to do deeper training.","metadata":{}},{"cell_type":"code","source":"sample_run = False\nif sample_run:\n    # need more in sample because so few records\n    vif_sample_frac = 0.1\nelse:\n    vif_sample_frac = 0.01","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:20.905512Z","iopub.execute_input":"2024-05-20T23:17:20.906241Z","iopub.status.idle":"2024-05-20T23:17:20.911384Z","shell.execute_reply.started":"2024-05-20T23:17:20.906192Z","shell.execute_reply":"2024-05-20T23:17:20.910554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl  # Similar to pandas, but with better performance for handling large datasets.\nimport pprint # PrettyPrinter() for dfs\nimport pandas as pd  # Library for importing CSV files.\nimport numpy as np  # Library for matrix operations.\nimport seaborn as sns\n# for VIF-based collinearity eliminator\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom glob import glob\nfrom pathlib import Path\n# Metric\nfrom sklearn.metrics import roc_auc_score  # Import the ROC-AUC curve.\n# KFold directly splits into k folds, while StratifiedKFold also considers class proportions.\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.decomposition import TruncatedSVD  # Truncated Singular Value Decomposition, a method for data dimensionality reduction.\nfrom sklearn.linear_model import LinearRegression\n# Chapter 9 of \"Finding Ghosts in Your Data\" by Kevin Feasel\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.impute import KNNImputer\nimport dill  # Serialization and deserialization of objects (e.g., saving and loading tree models).\nimport gc  # Garbage collection module.\nimport time  # Standard library time module.\n## For counting most frequent entries like the first 10 characters of predictors associated with the most outliers (see Counter below)\nfrom collections import Counter\n\nfrom pandas.core import base\nfrom statsmodels import robust\nimport matplotlib.pyplot as plt\n\n# Chapter 7 of \"Finding Ghosts in Your Data\"\nfrom scipy.stats import shapiro, normaltest, anderson, boxcox, zscore\n\nimport math","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:20.913354Z","iopub.execute_input":"2024-05-20T23:17:20.914073Z","iopub.status.idle":"2024-05-20T23:17:22.876426Z","shell.execute_reply.started":"2024-05-20T23:17:20.914033Z","shell.execute_reply":"2024-05-20T23:17:22.875049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pipeline","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int32))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\",\"pctinstlsallpaidlate4d_3546849L\",\"pctinstlsallpaidlate4d_3546849l\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.95:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if ((freq == 1) | (freq > 200) & (col != \"pctinstlsallpaidlate4d_3546849L\") & (col != \"pctinstlsallpaidlate4d_3546849l\")):\n                    df = df.drop(col)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.878022Z","iopub.execute_input":"2024-05-20T23:17:22.878476Z","iopub.status.idle":"2024-05-20T23:17:22.892804Z","shell.execute_reply.started":"2024-05-20T23:17:22.878448Z","shell.execute_reply":"2024-05-20T23:17:22.891198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Automatic aggregation","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n    \n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.894152Z","iopub.execute_input":"2024-05-20T23:17:22.895382Z","iopub.status.idle":"2024-05-20T23:17:22.911150Z","shell.execute_reply.started":"2024-05-20T23:17:22.895343Z","shell.execute_reply":"2024-05-20T23:17:22.909717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## File I/O","metadata":{}},{"cell_type":"code","source":"def read_file(path, depth=None):\n    \n    df = pl.read_parquet(path)\n    if sample_run:\n        if 'target' in df.columns and 'case_id' in df.columns:\n            df_target_1 = df.filter(df['target'] == 1).head(5) \n            df = df.sample(n=100, with_replacement=True)\n            df.extend(df_target_1)\n            df = df.unique(subset=['case_id'], maintain_order=True)        \n        \n    df = df.pipe(Pipeline.set_table_dtypes)\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        if sample_run:\n            if 'target' in df.columns and 'case_id' in df.columns:\n                df_target_1 = df.filter(df['target'] == 1).head(5) \n                df = df.sample(n=100, with_replacement=True)\n                df.extend(df_target_1)\n                df = df.unique(subset=['case_id'], maintain_order=True)\n            else:\n                df = df.sample(n=100, with_replacement=True)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        \n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n        chunks.append(df)\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.914146Z","iopub.execute_input":"2024-05-20T23:17:22.914588Z","iopub.status.idle":"2024-05-20T23:17:22.928759Z","shell.execute_reply.started":"2024-05-20T23:17:22.914547Z","shell.execute_reply":"2024-05-20T23:17:22.927211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configuration settings\nif sample_run:\n    class Config():\n        seed = 2024\n        num_folds = 3\n        TARGET_NAME = 'target'\n        batch_size = 333  # Since the test data size is unknown, we batch it into the model.\nelse:    \n    class Config():\n        seed = 2024\n        num_folds = 10\n        TARGET_NAME = 'target'\n        batch_size = 1000  # Since the test data size is unknown, we batch it into the model.\n\nimport random  # Provides functions for generating random numbers.\n# Set the random seed to ensure model reproducibility.\ndef seed_everything(seed):\n    np.random.seed(seed)  # Set numpy's random seed.\n    random.seed(seed)  # Set Python's built-in random seed.\nseed_everything(Config.seed)\n\n# Read the data types of each feature in the training data\ncolname2dtype = pd.read_csv(\"/kaggle/input/home-credit-inconsistent-data-types/colname2dtype.csv\")\ncolname = colname2dtype['Column'].values\ndtype = colname2dtype['DataType'].values\n\ndtype2pl = {}\ndtype2pl['Int64'] = pl.Int64\ndtype2pl['Float64'] = pl.Float64\ndtype2pl['String'] = pl.String\ndtype2pl['Boolean'] = pl.String\n\ncolname2dtype = {}\nfor idx in range(len(colname)):\n    colname2dtype[colname[idx]] = dtype2pl[dtype[idx]]\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.930242Z","iopub.execute_input":"2024-05-20T23:17:22.931136Z","iopub.status.idle":"2024-05-20T23:17:22.968657Z","shell.execute_reply.started":"2024-05-20T23:17:22.931093Z","shell.execute_reply":"2024-05-20T23:17:22.967628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    \n    if sample_run:\n        if 'target' in df_base.columns and 'case_id' in df_base.columns:\n                df_target_1 = df_base.filter(df_base['target'] == 1).head(5)\n                df_base = df_base.sample(n=100, with_replacement=True)\n                df_base.extend(df_target_1)\n                df_base = df_base.unique(subset=['case_id'], maintain_order=True)\n        else:\n            df_base = df_base.sample(n=100, with_replacement=True)\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    \n    return df_base","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.970286Z","iopub.execute_input":"2024-05-20T23:17:22.970946Z","iopub.status.idle":"2024-05-20T23:17:22.978452Z","shell.execute_reply.started":"2024-05-20T23:17:22.970911Z","shell.execute_reply":"2024-05-20T23:17:22.977567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.979498Z","iopub.execute_input":"2024-05-20T23:17:22.979824Z","iopub.status.idle":"2024-05-20T23:17:22.995867Z","shell.execute_reply.started":"2024-05-20T23:17:22.979798Z","shell.execute_reply":"2024-05-20T23:17:22.994932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:22.997394Z","iopub.execute_input":"2024-05-20T23:17:22.998072Z","iopub.status.idle":"2024-05-20T23:17:23.008982Z","shell.execute_reply.started":"2024-05-20T23:17:22.998033Z","shell.execute_reply":"2024-05-20T23:17:23.007876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:17:23.010312Z","iopub.execute_input":"2024-05-20T23:17:23.010689Z","iopub.status.idle":"2024-05-20T23:18:42.011467Z","shell.execute_reply.started":"2024-05-20T23:17:23.010661Z","shell.execute_reply":"2024-05-20T23:18:42.010045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nif \"pctinstlsallpaidlate4d_3546849L\" in df_train.columns:\n    print(\"The column 'pctinstlsallpaidlate4d_3546849L' exists in the DataFrame.\")\nelse:\n    print(\"The column 'pctinstlsallpaidlate4d_3546849L' does not exist in the DataFrame.\")\n    \nprint(\"train data shape:\\t\", df_train.shape) ","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.017356Z","iopub.execute_input":"2024-05-20T23:18:42.018246Z","iopub.status.idle":"2024-05-20T23:18:42.340744Z","shell.execute_reply.started":"2024-05-20T23:18:42.018207Z","shell.execute_reply":"2024-05-20T23:18:42.339436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to calculate percent missing for a single column\ndef calc_percent_missing(series, total):\n    null_count = series.null_count()\n    percent_missing = (null_count * 100) / total\n    return percent_missing\n\ndf_train.columns = [col.lower() for col in df_train.columns]\n\n# Check if any column name contains the string 'zip'\ncontains_zip_column = any('zip' in col for col in df_train.columns)\n\nprint(f\"Does any column name contain 'zip' after lower-casing? {contains_zip_column}\")\n\nzip_columns = [col for col in df_train.columns if 'zip' in col]\n\n# for col in zip_columns:\n#     total = len(df_train)\n    \n#     percent_missing = calc_percent_missing(df_train[col], total)\n#     print(col)\n#     new_series = pd.Series([percent_missing], name=col)\n#     percent_missing_zip = pd.concat([percent_missing_zip, new_series], how=\"vertical\")\n\ncount_missing_zip = (\n    df_train[list(zip_columns)]  # Convert zip_columns to a list\n    .null_count()\n    .to_pandas()\n)      \n\npercent_missing_zip = count_missing_zip / len(df_train)\nprint(\"Zip columns missing values\")\npprint.pprint(count_missing_zip)\nprint(\"Zip columns missing values - percent\")\npprint.pprint(percent_missing_zip)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.342286Z","iopub.execute_input":"2024-05-20T23:18:42.342637Z","iopub.status.idle":"2024-05-20T23:18:42.387566Z","shell.execute_reply.started":"2024-05-20T23:18:42.342588Z","shell.execute_reply":"2024-05-20T23:18:42.386632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.388752Z","iopub.execute_input":"2024-05-20T23:18:42.389273Z","iopub.status.idle":"2024-05-20T23:18:42.397018Z","shell.execute_reply.started":"2024-05-20T23:18:42.389246Z","shell.execute_reply":"2024-05-20T23:18:42.395596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top predictors based on the bottom of LGBM and Catboost https://www.kaggle.com/code/ondi1989/aa-final-lgbm-and-catboost-model\n#   These actually have different names likes totalamount_6A_sum so notebook likely derives based on what's given. \n#   Looking where least missing.\n#   dpd = days past due\ntop_preds = [col for col in df_train.columns if 'totalamount' in col or 'pmtnum' in col or 'incometype' in col or 'birth' in col or 'dpd' in col or 'mode_subjectrole_182M' in col]\n\n\n# for col in zip_columns:\n#     total = len(df_train)\n    \n#     percent_missing = calc_percent_missing(df_train[col], total)\n#     print(col)\n#     new_series = pd.Series([percent_missing], name=col)\n#     percent_missing_zip = pd.concat([percent_missing_zip, new_series], how=\"vertical\")\n\ncount_missing_top = (\n    df_train[list(top_preds)]  # Convert zip_columns to a list\n    .null_count()\n    .to_pandas()\n)      \n\npercent_missing_top = count_missing_top / len(df_train)\nprint(\"Top preds - LGBM/Catboost missing values\")\npprint.pprint(count_missing_top)\nprint(\"Top preds - LGBM/Catboost missing values - percent\")\npprint.pprint(percent_missing_top)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.398546Z","iopub.execute_input":"2024-05-20T23:18:42.399363Z","iopub.status.idle":"2024-05-20T23:18:42.428237Z","shell.execute_reply.started":"2024-05-20T23:18:42.399322Z","shell.execute_reply":"2024-05-20T23:18:42.427105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Files Read & Feature Engineering","metadata":{}},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.431978Z","iopub.execute_input":"2024-05-20T23:18:42.432341Z","iopub.status.idle":"2024-05-20T23:18:42.839302Z","shell.execute_reply.started":"2024-05-20T23:18:42.432315Z","shell.execute_reply":"2024-05-20T23:18:42.838074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\n\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.840786Z","iopub.execute_input":"2024-05-20T23:18:42.841213Z","iopub.status.idle":"2024-05-20T23:18:42.901157Z","shell.execute_reply.started":"2024-05-20T23:18:42.841175Z","shell.execute_reply":"2024-05-20T23:18:42.900049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to calculate percent missing for a single column\ndef calc_percent_missing(series, total):\n    null_count = series.null_count()\n    percent_missing = (null_count * 100) / total\n    return percent_missing\n\ndf_train.columns = [col.lower() for col in df_train.columns]\n\n# Check if any column name contains the string 'zip'\ncontains_zip_column = any('zip' in col for col in df_train.columns)\n\nprint(f\"Does any column name contain 'zip' after lower-casing? {contains_zip_column}\")\n\nzip_columns = [col for col in df_train.columns if 'zip' in col]\n\n# for col in zip_columns:\n#     total = len(df_train)\n    \n#     percent_missing = calc_percent_missing(df_train[col], total)\n#     print(col)\n#     new_series = pd.Series([percent_missing], name=col)\n#     percent_missing_zip = pd.concat([percent_missing_zip, new_series], how=\"vertical\")\n\ncount_missing_zip = (\n    df_train[list(zip_columns)]  # Convert zip_columns to a list\n    .null_count()\n    .to_pandas()\n)      \n\npercent_missing_zip = count_missing_zip / len(df_train)\nprint(\"Zip columns missing values\")\npprint.pprint(count_missing_zip)\nprint(\"Zip columns missing values - percent\")\npprint.pprint(percent_missing_zip)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.902120Z","iopub.execute_input":"2024-05-20T23:18:42.902415Z","iopub.status.idle":"2024-05-20T23:18:42.917579Z","shell.execute_reply.started":"2024-05-20T23:18:42.902391Z","shell.execute_reply":"2024-05-20T23:18:42.916232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(df_train)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.919590Z","iopub.execute_input":"2024-05-20T23:18:42.920056Z","iopub.status.idle":"2024-05-20T23:18:42.939274Z","shell.execute_reply.started":"2024-05-20T23:18:42.920012Z","shell.execute_reply":"2024-05-20T23:18:42.938056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"late_payment_columns = [col for col in df_train.columns if 'pctinstlsallpaidlate' in col]\nprint(late_payment_columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.940896Z","iopub.execute_input":"2024-05-20T23:18:42.941347Z","iopub.status.idle":"2024-05-20T23:18:42.950293Z","shell.execute_reply.started":"2024-05-20T23:18:42.941300Z","shell.execute_reply":"2024-05-20T23:18:42.949197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_missing_zip = (\n    df_train[list(zip_columns)]  # Convert zip_columns to a list\n    .null_count()\n    .to_pandas()\n)      \n\npercent_missing_zip = count_missing_zip / len(df_train)\n\npprint.pprint(percent_missing_zip)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.951398Z","iopub.execute_input":"2024-05-20T23:18:42.951787Z","iopub.status.idle":"2024-05-20T23:18:42.967379Z","shell.execute_reply.started":"2024-05-20T23:18:42.951758Z","shell.execute_reply":"2024-05-20T23:18:42.966210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Elimination\nLet's also examine common first letters of features being eliminated vs. those in the full df_train","metadata":{}},{"cell_type":"code","source":"df_train = df_train.pipe(Pipeline.filter_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:42.968646Z","iopub.execute_input":"2024-05-20T23:18:42.969315Z","iopub.status.idle":"2024-05-20T23:18:43.114062Z","shell.execute_reply.started":"2024-05-20T23:18:42.969277Z","shell.execute_reply":"2024-05-20T23:18:43.112812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filter_cols = set(df_train.columns) - set(df_test.columns)\nif 'target' in filter_cols:\n    filter_cols.discard('target')\nif 'month_decision' not in filter_cols:\n    filter_cols.add('month_decision')\n\n# Get the first 10 characters of each unique value\nfirst_10_chars = [val[:10] for val in df_train.columns]\nfirst_10_chars_filter_cols = [val[:10] for val in filter_cols]\n\n# Count the frequencies of each 10-character string\ncounter = Counter(first_10_chars)\nall_columns_first_10_chars = counter.most_common()\n\ncounter_filter_cols = Counter(first_10_chars_filter_cols)\n\nrank_dict = {k: rank+1 for rank, (k, v) in enumerate(all_columns_first_10_chars)}\n\n\n# Calculate total number of records\ntotal_records = len(first_10_chars)\ntotal_records_filter_cols = len(first_10_chars_filter_cols)\n\n# Get the most common 10-character strings, sorted by frequency\n# with percentage of those column prefixes found in the filter_cols df then entire df\nmost_common_filter_cols = [(k, v, v/total_records_filter_cols*100, rank_dict.get(k, 'N/A'), i+1) for i, (k, v) in enumerate(counter_filter_cols.most_common(20))]\n\ndf_most_common_filter_cols = pd.DataFrame(most_common_filter_cols, columns=['Col_Pre', 'Outlier_Freq', 'Outlier_Col_Pct', 'All_Data_Rank', 'Outlier_Rank'])\n\npprint.pprint(df_most_common_filter_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.115759Z","iopub.execute_input":"2024-05-20T23:18:43.116198Z","iopub.status.idle":"2024-05-20T23:18:43.133187Z","shell.execute_reply.started":"2024-05-20T23:18:43.116159Z","shell.execute_reply":"2024-05-20T23:18:43.131863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.to_pandas()\ndf_test  = df_test.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.135199Z","iopub.execute_input":"2024-05-20T23:18:43.135543Z","iopub.status.idle":"2024-05-20T23:18:43.163992Z","shell.execute_reply.started":"2024-05-20T23:18:43.135514Z","shell.execute_reply":"2024-05-20T23:18:43.162693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.columns = [col.lower() for col in df_test.columns]","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.165734Z","iopub.execute_input":"2024-05-20T23:18:43.167402Z","iopub.status.idle":"2024-05-20T23:18:43.173313Z","shell.execute_reply.started":"2024-05-20T23:18:43.167366Z","shell.execute_reply":"2024-05-20T23:18:43.172152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## No NA can appear for a column!\nAlternative would be imputation.","metadata":{}},{"cell_type":"code","source":"df_train = df_train.dropna(axis=1, how='any')","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.174573Z","iopub.execute_input":"2024-05-20T23:18:43.174979Z","iopub.status.idle":"2024-05-20T23:18:43.191815Z","shell.execute_reply.started":"2024-05-20T23:18:43.174948Z","shell.execute_reply":"2024-05-20T23:18:43.190214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_keep = [col for col in df_train.columns if col not in [\"target\"]]\ndf_test = df_test[columns_to_keep]\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.193840Z","iopub.execute_input":"2024-05-20T23:18:43.194329Z","iopub.status.idle":"2024-05-20T23:18:43.203589Z","shell.execute_reply.started":"2024-05-20T23:18:43.194291Z","shell.execute_reply":"2024-05-20T23:18:43.202494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform one-hot encoding transformation on string feature columns\nprint(\"----------string one hot encoder ****\")\nfor col in df_train.columns:\n    n_unique=df_train[col].nunique()\n    # If it's a categorical variable, perform one-hot encoding\n    # If the category is binary, like gender, if it's (0,1) or numeric type, there's no need to transform. If it's a string type, transform it into numeric\n    if n_unique==2 and df_train[col].dtype=='object':\n        print(f\"one_hot_2:{col}\")\n        unique=df_train[col].unique()\n        # Choose any category for transformation, for example, gender='Female'\n        df_train[col]=(df_train[col]==unique[0]).astype(int)\n        df_test[col]=(df_test[col]==unique[0]).astype(int)\n    elif (n_unique<10) and df_train[col].dtype=='object': # Due to limited memory, the n_unique of categorical variables is set to 10\n        print(f\"one_hot_10:{col}\")\n        unique=df_train[col].unique()\n        for idx in range(len(unique)):\n            if unique[idx]==unique[idx]: # This is to avoid the situation where there are nan values in the string\n                df_train[col+\"_\"+str(idx)]=(df_train[col]==unique[idx]).astype(int)\n                df_test[col+\"_\"+str(idx)]=(df_test[col]==unique[idx]).astype(int)\n        df_train.drop([col],axis=1,inplace=True)\n        df_test.drop([col],axis=1,inplace=True)\nprint(\"----------drop other string or unique value or full null value ****\")\ndrop_cols=[]\nfor col in df_test.columns:\n    if (df_train[col].dtype=='object') or (df_test[col].dtype=='object') \\\n        or (df_train[col].nunique()==1) or df_train[col].isna().mean()>0.99:\n        drop_cols+=[col]\n# 'case_id' is not very useful.\ndrop_cols+=['case_id']\ndf_train.drop(drop_cols,axis=1,inplace=True)\ndf_test.drop(drop_cols,axis=1,inplace=True)\nprint(f\"len(df_train):{len(df_train)},total_features_counts:{len(df_test.columns)}\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.205176Z","iopub.execute_input":"2024-05-20T23:18:43.205507Z","iopub.status.idle":"2024-05-20T23:18:43.272898Z","shell.execute_reply.started":"2024-05-20T23:18:43.205480Z","shell.execute_reply":"2024-05-20T23:18:43.272059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Traverse all columns of the dataframe df to change data types and reduce memory usage\ndef reduce_mem_usage(df, float16_as32=True):\n    # memory_usage() is the memory usage of each column of df, sum is their sum, B->KB->MB\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns: # Traverse each column name\n        col_type = df[col].dtype # Type of the column\n        if col_type != object: # Not an object, i.e., here we are dealing with numeric variables\n            c_min,c_max = df[col].min(),df[col].max() # Find the minimum and maximum of this column\n            if str(col_type)[:3] == 'int': # If it's an integer type variable, regardless of int8, int16, int32 or int64\n                # If the range of this column is within the range of int8, then convert the type (-128 to 127)\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                # If the range of this column is within the range of int16, then convert the type (-32,768 to 32,767)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                # If the range of this column is within the range of int32, then convert the type (-2,147,483,648 to 2,147,483,647)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                # If the range of this column is within the range of int64, then convert the type (-9,223,372,036,854,775,808 to 9,223,372,036,854,775,807)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else: # If it's a floating point type.\n                # If the value is within the range of float16, consider float32 if higher precision is needed\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32: # If higher precision is needed, choose float32\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float16)  \n                # If the value is within the range of float32, convert its type\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                # If the value is within the range of float64, convert its type\n                else:\n                    df[col] = df[col].astype(np.float64)\n    # Calculate the memory after the operation\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    # How much percentage has the memory decreased compared to the beginning\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\ndf_train = reduce_mem_usage(df_train)\ndf_test = reduce_mem_usage(df_test)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.274058Z","iopub.execute_input":"2024-05-20T23:18:43.274582Z","iopub.status.idle":"2024-05-20T23:18:43.317820Z","shell.execute_reply.started":"2024-05-20T23:18:43.274552Z","shell.execute_reply":"2024-05-20T23:18:43.316837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_columns = df_train.shape[1]\nprint(f\"The number of columns in df_train is {num_columns}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.324444Z","iopub.execute_input":"2024-05-20T23:18:43.324983Z","iopub.status.idle":"2024-05-20T23:18:43.334723Z","shell.execute_reply.started":"2024-05-20T23:18:43.324946Z","shell.execute_reply":"2024-05-20T23:18:43.333394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Ran out of memory attempting to use loci for outlier detection.\n## Code in version 24","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.336302Z","iopub.execute_input":"2024-05-20T23:18:43.336651Z","iopub.status.idle":"2024-05-20T23:18:43.346517Z","shell.execute_reply.started":"2024-05-20T23:18:43.336603Z","shell.execute_reply":"2024-05-20T23:18:43.345226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the correlation of each feature with the target\ncorrelations = df_train.corr()['target'].apply(abs).sort_values(ascending=False)\n\n# Get the top 20 most correlated features along with the target\nif sample_run:\n    top_corr_features = correlations.index[:5]\nelse:\n    top_corr_features = correlations.index[:20]\n\n# Create a new dataframe with the most correlated features\ndf_train_most_corr_20_target = df_train[top_corr_features]\n\n# Print the correlations in a table\nprint(correlations.head(21))\n\n# Plot the correlations\nplt.figure(figsize=(12, 10))\nsns.heatmap(df_train_most_corr_20_target.corr(), annot=True, cmap='coolwarm')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.348337Z","iopub.execute_input":"2024-05-20T23:18:43.349251Z","iopub.status.idle":"2024-05-20T23:18:43.881067Z","shell.execute_reply.started":"2024-05-20T23:18:43.349215Z","shell.execute_reply":"2024-05-20T23:18:43.879688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# No feature is normally distributed\nHad to use sample here because running out of RAM. Repeated attempts supported no predictor was normally distributed.\n\nThis affects what methods can be used for anomaly detection.","metadata":{}},{"cell_type":"code","source":"# Initialize lists to hold the feature names\nnormal_features = []\nnot_normal_features = []\n\n# Define the sample size\nsample_size = min(100000, len(df_train))\n\n# Iterate over the top 20 correlated features\nfor feature in top_corr_features:\n    # Get a sample of the data for the current feature\n    data_sample = df_train[feature].sample(sample_size, random_state=0).dropna()\n\n    # Perform the D'Agostino's K^2^ test for normality\n    k2, p = normaltest(data_sample)\n\n    # If p < 0.05, the null hypothesis is rejected and the distribution is not normal\n    if p < 0.05:\n        not_normal_features.append(feature)\n    else:\n        normal_features.append(feature)\n\n# Print the results\nprint(\"Normal features:\", normal_features)\nprint(\"Not normal features:\", not_normal_features)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.882572Z","iopub.execute_input":"2024-05-20T23:18:43.883072Z","iopub.status.idle":"2024-05-20T23:18:43.915302Z","shell.execute_reply.started":"2024-05-20T23:18:43.883036Z","shell.execute_reply":"2024-05-20T23:18:43.914113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the top 10 most correlated features along with the target\ntop_10_corr_features = correlations.index[:11]\n\n# Create a new dataframe with the most correlated features\ndf_train_most_corr_10_target = df_train[top_10_corr_features]\n\n# Plot the correlations\nplt.figure(figsize=(12, 10))\nsns.heatmap(df_train_most_corr_10_target.corr(), annot=True, cmap='coolwarm')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:43.916635Z","iopub.execute_input":"2024-05-20T23:18:43.917027Z","iopub.status.idle":"2024-05-20T23:18:44.598993Z","shell.execute_reply.started":"2024-05-20T23:18:43.916997Z","shell.execute_reply":"2024-05-20T23:18:44.597981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_corr_features","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.601067Z","iopub.execute_input":"2024-05-20T23:18:44.601412Z","iopub.status.idle":"2024-05-20T23:18:44.607360Z","shell.execute_reply.started":"2024-05-20T23:18:44.601377Z","shell.execute_reply":"2024-05-20T23:18:44.606289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extreme quantile-based outlier elimination\nKNN below may be used later. Would need to use sampling.\nThe target = 1 case is <3% of records as I recall so only looking for quantile outliers where target = 0.","metadata":{}},{"cell_type":"code","source":"# Calculate the quantiles (1% and 99%)\ndf_train['base_row_number'] = range(len(df_train))\ndf_train['base_row_number'] = df_train['base_row_number'] + 1\ntarget0_df = df_train[df_train['target'] == 0]\nq_low = target0_df.quantile(0.001)\nq_high = target0_df.quantile(0.999)\n\n# For finding lower and upper outliers\nlow_iter  = 0\nhigh_iter = 0\n# Create a mask for each column\ncolumn_masks = {}\nrow_numbers_outliers_target0 = []\nunique_outlier_cols = []\n\nfor col in df_train.columns:\n    if col in q_low and col in q_high:\n        if col != 'base_row_number':\n            column_masks[col] = (target0_df[col] < q_low[col]) | (target0_df[col] > q_high[col])\n            if column_masks[col].any():\n                unique_outlier_cols.append(col)\nunique_outlier_cols = list(set(unique_outlier_cols))\n\nfor row_index, row in target0_df.iterrows():\n    condition = pd.Series([False] * len(row), index=row.index)\n    if col != 'base_row_number':\n        # Check if any column value exceeds the 10th quartile\n        condition = pd.Series([False] * len(row), index=row.index)\n    for col in row.index:\n        if col != 'base_row_number':\n            condition[col] = (row[col] < q_low[col]) | (row[col] > q_high[col])\n    if condition.any():\n        row_numbers_outliers_target0.append(row['base_row_number'])\n        for col in row.index:\n            if col != 'base_row_number':\n                if row[col] < q_low[col]:\n                        low_iter+=1\n                        if low_iter < 3:                        \n                            print(f\"Example lower outlier for column {col}: {row[col]}, q_low: {q_low[col]}\")\n                            break\n                if row[col] > q_high[col]:\n                        high_iter+=1\n                        if high_iter < 3:                        \n                            print(f\"Example upper outlier for column {col}: {row[col]}, q_high: {q_high[col]}\")\n                            break\n\nmask = df_train['base_row_number'].isin(row_numbers_outliers_target0)\ndf_train.drop('base_row_number', axis=1, inplace=True)\n\neliminated_records = df_train.loc[mask]\n# Drop the rows based on the mask\ndf_train.drop(df_train[mask].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.609298Z","iopub.execute_input":"2024-05-20T23:18:44.609810Z","iopub.status.idle":"2024-05-20T23:18:44.705440Z","shell.execute_reply.started":"2024-05-20T23:18:44.609657Z","shell.execute_reply":"2024-05-20T23:18:44.704216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Target distribution for outliers","metadata":{}},{"cell_type":"code","source":"relative_frequency_target_outliers = eliminated_records['target'].value_counts(normalize=True)\nrelative_frequency_target = df_train['target'].value_counts(normalize=True)\nprint(relative_frequency_target_outliers)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.706851Z","iopub.execute_input":"2024-05-20T23:18:44.707259Z","iopub.status.idle":"2024-05-20T23:18:44.715695Z","shell.execute_reply.started":"2024-05-20T23:18:44.707225Z","shell.execute_reply":"2024-05-20T23:18:44.714493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Target distribution for all data","metadata":{}},{"cell_type":"code","source":"print(relative_frequency_target)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.717553Z","iopub.execute_input":"2024-05-20T23:18:44.718717Z","iopub.status.idle":"2024-05-20T23:18:44.729073Z","shell.execute_reply.started":"2024-05-20T23:18:44.718646Z","shell.execute_reply":"2024-05-20T23:18:44.728020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unique columns in the quantile outlier test","metadata":{}},{"cell_type":"code","source":"unique_values_outliers = sorted(unique_outlier_cols)\n\n# Get the first 10 characters of each unique value\nfirst_10_chars = [val[:10] for val in df_train.columns]\nfirst_10_chars_outliers = [val[:10] for val in unique_values_outliers]\n\n# Count the frequencies of each 10-character string\ncounter = Counter(first_10_chars)\nall_columns_first_10_chars = counter.most_common()\n\ncounter_outliers = Counter(first_10_chars_outliers)\n\nrank_dict = {k: rank+1 for rank, (k, v) in enumerate(all_columns_first_10_chars)}\n\n\n# Calculate total number of records\ntotal_records = len(first_10_chars)\ntotal_records_outliers = len(first_10_chars_outliers)\n\n# Get the most common 10-character strings, sorted by frequency\n# with percentage of those column prefixes found in the outliers df then entire df\nmost_common_outliers = [(k, v, v/total_records_outliers*100, rank_dict.get(k, 'N/A'), i+1) for i, (k, v) in enumerate(counter_outliers.most_common(20))]\n\ndf_most_common_outliers = pd.DataFrame(most_common_outliers, columns=['Col_Pre', 'Outlier_Freq', 'Outlier_Col_Pct', 'All_Data_Rank', 'Outlier_Rank'])\n\npprint.pprint(df_most_common_outliers)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.730500Z","iopub.execute_input":"2024-05-20T23:18:44.731783Z","iopub.status.idle":"2024-05-20T23:18:44.746358Z","shell.execute_reply.started":"2024-05-20T23:18:44.731750Z","shell.execute_reply":"2024-05-20T23:18:44.744876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# knn_outlier_test not completing, will change this code to a more interesting test than Z-score outlier\n\n# This line would require internet to be on and installing pyod\n# \"Internet on\" eliminates ability to submit for this competition\n\n# from pyod.models.knn import KNN\n# \n# import numpy as np\n# import pandas as pd\n# from pyod.models.knn import KNN\n# \n# def knn_outlier_test(df):\n#     results = []\n#     for column in df.select_dtypes(include=[np.number]).columns:\n#         data = df[column].values.reshape(-1, 1)\n#         # n_jobs=-1 for parallel processing\n#         clf = KNN(contamination=0.02, n_jobs=-1)  # initialize detector\n#         clf.fit(data)  # fit the model\n#         outliers = clf.predict(data)  # get the prediction labels of the training data\n#         outlier_indexes = np.where(outliers == 1)[0]  # get the outlier indices\n#         \n#         for outlier_index in outlier_indexes:\n#             results.append((column, 'KNN', outlier_index, data[outlier_index][0]))\n#     \n#     return pd.DataFrame(results, columns=['Column', 'Test Used', 'Observation', 'Outlier Value'])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.747954Z","iopub.execute_input":"2024-05-20T23:18:44.748310Z","iopub.status.idle":"2024-05-20T23:18:44.755632Z","shell.execute_reply.started":"2024-05-20T23:18:44.748283Z","shell.execute_reply":"2024-05-20T23:18:44.754690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Key parameter to adjust : Pearson Correlation for Linear Regression\n\nAdjust number in this line below to easily use different variables. Experiment to see how this affects the score when you submit.\n\n> if abs(pearson)>0.0020:","metadata":{}},{"cell_type":"code","source":"def pearson_corr(x1,x2):\n    \"\"\"\n    x1,x2:np.array\n    \"\"\"\n    mean_x1=np.mean(x1)\n    mean_x2=np.mean(x2)\n    std_x1=np.std(x1)\n    std_x2=np.std(x2)\n    pearson=np.mean((x1-mean_x1)*(x2-mean_x2))/(std_x1*std_x2)\n    return pearson\n\n# Are there any features that are particularly highly correlated with the target? Use them for logistic regression.\nchoose_cols=[]\nall_correlations = {}  # Create a dictionary to store all correlations\n\nfor col in df_train.columns:\n    if col != 'target':\n        pearson=pearson_corr(df_train[col].values,df_train['target'].values) \n        all_correlations[col] = pearson  # Store the correlation in the dictionary\n        # .0025 yielded score of 0.5 in early version of notebook\n        if abs(pearson)>0.0024:\n            choose_cols.append(col)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.757026Z","iopub.execute_input":"2024-05-20T23:18:44.757487Z","iopub.status.idle":"2024-05-20T23:18:44.778102Z","shell.execute_reply.started":"2024-05-20T23:18:44.757462Z","shell.execute_reply":"2024-05-20T23:18:44.776711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=df_train[choose_cols].copy()\ny=df_train[Config.TARGET_NAME].copy()\n\ntest_X=df_test[choose_cols].copy()\ndel df_train,df_test\ngc.collect() # This line manually triggers garbage collection to forcibly reclaim memory that the garbage collector has marked as unused.","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.779675Z","iopub.execute_input":"2024-05-20T23:18:44.781051Z","iopub.status.idle":"2024-05-20T23:18:44.908837Z","shell.execute_reply.started":"2024-05-20T23:18:44.781000Z","shell.execute_reply":"2024-05-20T23:18:44.907533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using top predictors in KNN for imputing missing values\nLinearRegression() fails if any NaN","metadata":{}},{"cell_type":"code","source":"# Get correlation scores between each feature and target\ncorr_scores = mutual_info_regression(X, y)\n\n# Get all predictors\nall_predictors = X.columns\n\nimputer = KNNImputer(n_neighbors=5, weights=\"uniform\", metric=\"nan_euclidean\")\nX_imputed = imputer.fit_transform(X[all_predictors])\nX_full = X.copy()\nX_full[all_predictors] = X_imputed\n\ntest_X_imputed = imputer.transform(test_X[all_predictors])\ntest_X_full = test_X.copy()\ntest_X_full[all_predictors] = test_X_imputed\n\nX = X_full\ntest_X = test_X_full\n\ndel X_full, test_X_full","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.910250Z","iopub.execute_input":"2024-05-20T23:18:44.910591Z","iopub.status.idle":"2024-05-20T23:18:44.971540Z","shell.execute_reply.started":"2024-05-20T23:18:44.910561Z","shell.execute_reply":"2024-05-20T23:18:44.970642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Missing value check","metadata":{}},{"cell_type":"code","source":"percent_missing = X.isnull().mean() * 100\nmissing_value_df = pd.DataFrame({'column_name': X.columns, 'percent_missing': percent_missing})\nmissing_value_df.sort_values('percent_missing', inplace=True)\n\nwith pd.option_context('display.max_rows', None, 'display.max_columns', None):\n    print(missing_value_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.972887Z","iopub.execute_input":"2024-05-20T23:18:44.973872Z","iopub.status.idle":"2024-05-20T23:18:44.985384Z","shell.execute_reply.started":"2024-05-20T23:18:44.973837Z","shell.execute_reply":"2024-05-20T23:18:44.984424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"percent_missing = test_X.isnull().mean() * 100\nmissing_value_df = pd.DataFrame({'column_name': test_X.columns, 'percent_missing': percent_missing})\nmissing_value_df.sort_values('percent_missing', inplace=True)\n\nprint(missing_value_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:18:44.986817Z","iopub.execute_input":"2024-05-20T23:18:44.987761Z","iopub.status.idle":"2024-05-20T23:18:45.004953Z","shell.execute_reply.started":"2024-05-20T23:18:44.987730Z","shell.execute_reply":"2024-05-20T23:18:45.003499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Drop predictors - multicolinearity based on VIF","metadata":{}},{"cell_type":"code","source":"# Sampling because VIF ran out of memory\ndef calculate_vif_drop_common_columns(X, num_iterations=3, thresh=10, seed=39):\n    common_columns = set()  # Initialize an empty set\n\n    for _ in range(num_iterations):   \n        print(\"Iteration \" + str(_))     \n        sampled_df = X.sample(frac=vif_sample_frac, random_state=seed)               \n               \n        columns_to_drop = []\n        if sampled_df.empty:\n          continue  # Skip this iteration if dataframe is empty (sample_run may be True)\n        else:\n          # Calculate VIF for each column\n          vif = [variance_inflation_factor(sampled_df.values, ix)\n               for ix in range(sampled_df.shape[1])]\n          columns_to_drop = [sampled_df.columns[ix] for ix, vif_value in enumerate(vif) if vif_value > thresh]\n        \n        # This causes the first iteration to provide the columns that must then appear in all other iterations for it to be dropped.\n        if not common_columns:\n            common_columns.update(columns_to_drop)\n        else:\n            common_columns.intersection_update(columns_to_drop)\n\n    # Drop columns that appear in each iteration of VIF\n    X_dropped = X.drop(columns=common_columns)\n\n    return X_dropped","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:24:28.509672Z","iopub.execute_input":"2024-05-20T23:24:28.510539Z","iopub.status.idle":"2024-05-20T23:24:28.518549Z","shell.execute_reply.started":"2024-05-20T23:24:28.510498Z","shell.execute_reply":"2024-05-20T23:24:28.517668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = calculate_vif_drop_common_columns(X)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:24:42.952336Z","iopub.execute_input":"2024-05-20T23:24:42.952757Z","iopub.status.idle":"2024-05-20T23:24:42.966907Z","shell.execute_reply.started":"2024-05-20T23:24:42.952728Z","shell.execute_reply":"2024-05-20T23:24:42.965325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Linear Regression 1","metadata":{}},{"cell_type":"code","source":"#Prior to VF and Quantile usage\n#mean_gini:0.5428968427934477 with abs(pearson)>0.0025? Might check\n#mean_gini:0.5415694487332248 with abs(pearson)>0.0024\n#When before VIF but was in code mean_gini = 4.40 PB = 0.318\noof_pred_pro=np.zeros((len(X)))\ntest_pred_pro=np.zeros((Config.num_folds,len(test_X)))\n\n# Get training data feature names\ntrain_features = X.columns\n\n# Filter test_X to keep only matching columns\ntest_X = test_X[train_features]\n\nskf = StratifiedKFold(n_splits=Config.num_folds,random_state=Config.seed, shuffle=True)\n\nfor fold, (train_index, valid_index) in (enumerate(skf.split(X, y.astype(str)))):\n    print(f\"fold:{fold}\")\n\n    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n\n    # “create a linear regression model”.\n    model = LinearRegression()\n    model.fit(X_train,y_train)\n\n    oof_pred_pro[valid_index]=model.predict(X_valid)\n    # “predict the data in batches”.\n    for idx in range(0,len(test_X),Config.batch_size):\n        test_pred_pro[fold][idx:idx+Config.batch_size]=model.predict(test_X[idx:idx+Config.batch_size]) \n    del model,X_train, X_valid,y_train, y_valid #This line deletes the model and the training and validation sets after they have been used.\n    gc.collect() #This line manually triggers garbage collection to forcibly reclaim memory that the garbage collector has marked as unused.\ngini=2*roc_auc_score(y.values,oof_pred_pro)-1\nprint(f\"mean_gini:{gini}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:03.169269Z","iopub.execute_input":"2024-05-20T23:25:03.169710Z","iopub.status.idle":"2024-05-20T23:25:04.161764Z","shell.execute_reply.started":"2024-05-20T23:25:03.169679Z","shell.execute_reply":"2024-05-20T23:25:04.160678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_model = LinearRegression()\nfull_model.fit(X,y)\n\ny_pred = full_model.predict(X)\n\ndef plot_linear_model_coefficients(coefficients, columns, model_name):\n    feature_importance = np.array(coefficients)\n    feature_names = np.array(columns)\n\n    data = {'feature_names': feature_names, 'feature_importance': feature_importance}\n    temp_df = pd.DataFrame(data)\n\n    temp_df.sort_values(by=['feature_importance'], ascending=False, inplace=True)\n\n    plt.figure(figsize=(20, 20))\n    # Plot only top 20 features\n    sns.barplot(x=temp_df['feature_importance'][:20], y=temp_df['feature_names'][:20])\n\n    plt.title(model_name)\n    plt.xlabel('Feature Importance')\n    plt.ylabel('Feature Names')\n\n\nplot_linear_model_coefficients(np.abs(full_model.coef_), X.columns, 'Linear Regression Coefficients (Predictor Importance)')","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:04.163681Z","iopub.execute_input":"2024-05-20T23:25:04.164034Z","iopub.status.idle":"2024-05-20T23:25:04.885242Z","shell.execute_reply.started":"2024-05-20T23:25:04.164005Z","shell.execute_reply":"2024-05-20T23:25:04.884300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds=test_pred_pro.mean(axis=0)\nsubmission=pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\n\nif sample_run or len(test_preds) != submission.shape[0]:\n    print(\"Tripped\")\n    # Create a new DataFrame with the desired number of rows\n    new_submission = pd.DataFrame({'score': np.zeros(len(submission))})\n\n    # Copy the necessary columns from the original submission DataFrame\n    new_submission = new_submission.join(submission.drop('score', axis=1))\n\n    # Assign the test_preds values to the 'score' column\n    new_submission['score'] = np.clip(np.nan_to_num(test_preds[:len(submission)], nan=0.3), 0, 1)\n\n    submission = new_submission\nelse:\n    submission['score']=np.clip(np.nan_to_num(test_preds,nan=0.3),0,1)\nsubmission = submission[['case_id','score']]    \n\n# submission['case_id'] = submission['case_id'].round().astype(int)\nsubmission.to_csv(\"submission.csv\",index=None)\nprint(submission)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:04.886357Z","iopub.execute_input":"2024-05-20T23:25:04.887252Z","iopub.status.idle":"2024-05-20T23:25:04.913298Z","shell.execute_reply.started":"2024-05-20T23:25:04.887219Z","shell.execute_reply":"2024-05-20T23:25:04.912110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluating relative frequency of target : submission vs. training data","metadata":{}},{"cell_type":"code","source":"relative_frequency_submission = submission['score'].value_counts(normalize=True)\nrelative_frequency_submission_rounded = round(submission['score']).value_counts(normalize=True)\n## Defined above, but df_train dropped\nrelative_frequency_target = y.value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:04.915542Z","iopub.execute_input":"2024-05-20T23:25:04.915920Z","iopub.status.idle":"2024-05-20T23:25:04.926996Z","shell.execute_reply.started":"2024-05-20T23:25:04.915892Z","shell.execute_reply":"2024-05-20T23:25:04.925693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create dot plot\nplt.figure(figsize=(10, 5))\nplt.plot(relative_frequency_submission, 'o')\n\n# Set labels\nplt.xlabel('Score')\nplt.ylabel('Relative Frequency')\nplt.title('Dot Plot of Relative Frequency of Scores')\n\n# Show the plot\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:04.928607Z","iopub.execute_input":"2024-05-20T23:25:04.929078Z","iopub.status.idle":"2024-05-20T23:25:05.230106Z","shell.execute_reply.started":"2024-05-20T23:25:04.929048Z","shell.execute_reply":"2024-05-20T23:25:05.228975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(relative_frequency_submission_rounded)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:05.231689Z","iopub.execute_input":"2024-05-20T23:25:05.232126Z","iopub.status.idle":"2024-05-20T23:25:05.238975Z","shell.execute_reply.started":"2024-05-20T23:25:05.232088Z","shell.execute_reply":"2024-05-20T23:25:05.237685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(relative_frequency_target)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T23:25:05.240689Z","iopub.execute_input":"2024-05-20T23:25:05.241151Z","iopub.status.idle":"2024-05-20T23:25:05.251422Z","shell.execute_reply.started":"2024-05-20T23:25:05.241113Z","shell.execute_reply":"2024-05-20T23:25:05.250200Z"},"trusted":true},"execution_count":null,"outputs":[]}]}