{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":166996856,"sourceType":"kernelVersion"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# English comments for the notebook created by yunsuxiaozi 2024/3/20\n## EDA Additions made at bottom for \"Kaggle Data Outliers Using R and Python\" presentation on Tuesday, March 26, 2024 https://www.meetup.com/tripass/events/299619548\n\nComments otherwise are by Kaggle user yunsuxiaozi, copied from their notebook.\nOutlier detection work inspired by Kevin Feasel's 2013 book \"Finding Ghosts in Your Data\", and its Github repo at: https://github.com/Apress/finding-ghosts-in-your-data/blob/06ec44ebc8d69ab1ec018fb7d92a565ec2cb065b/code/src/app/models/univariate.py#L50\n\n### Why did linear regression perform well in this competition?\n\n### IMO,The main consideration for this competition is still the stability of the model. Although the linear regression model \"mean_gini\" is not good, it has good stability; lightgbm works well on \"mean_gini\", its stability is not good. This is why there was not much difference in their scores in this competition.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Tweak sample_run parameter when ready to do deeper training.","metadata":{}},{"cell_type":"code","source":"sample_run = True","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:10.993899Z","iopub.execute_input":"2024-12-09T21:37:10.994406Z","iopub.status.idle":"2024-12-09T21:37:11.034593Z","shell.execute_reply.started":"2024-12-09T21:37:10.994364Z","shell.execute_reply":"2024-12-09T21:37:11.032163Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl  # Similar to pandas, but with better performance for handling large datasets.\nimport pprint # PrettyPrinter() for dfs\nimport pandas as pd  # Library for importing CSV files.\nimport numpy as np  # Library for matrix operations.\nimport seaborn as sns\n# for VIF-based collinearity eliminator\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n# Metric\nfrom sklearn.metrics import roc_auc_score  # Import the ROC-AUC curve.\n# KFold directly splits into k folds, while StratifiedKFold also considers class proportions.\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.decomposition import TruncatedSVD  # Truncated Singular Value Decomposition, a method for data dimensionality reduction.\nimport dill  # Serialization and deserialization of objects (e.g., saving and loading tree models).\nimport gc  # Garbage collection module.\nimport time  # Standard library time module.\n## For counting most frequent entries like the first 10 characters of predictors associated with the most outliers (see Counter below)\nfrom collections import Counter\n\nfrom pandas.core import base\nfrom statsmodels import robust\nimport matplotlib.pyplot as plt\n\n\n# Chapter 7 of \"Finding Ghosts in Your Data\"\nfrom scipy.stats import shapiro, normaltest, anderson, boxcox, zscore\n\nimport math\n# Chapter 9 of \"Finding Ghosts in Your Data\"\nfrom sklearn.mixture import GaussianMixture\n# To ensure that when calling the trained model later, the correct version is used,\n# provide the time of model training.\n# The time.strftime() function formats a time object into a string,\n# and time.localtime() returns the current local time as a time.struct_time object.\ncurrent_time = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\nprint(\"This notebook's training time is\", current_time)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:11.044459Z","iopub.execute_input":"2024-12-09T21:37:11.044838Z","iopub.status.idle":"2024-12-09T21:37:15.072562Z","shell.execute_reply.started":"2024-12-09T21:37:11.044809Z","shell.execute_reply":"2024-12-09T21:37:15.071198Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Configuration settings\nif sample_run:\n    class Config():\n        seed = 2024\n        num_folds = 3\n        TARGET_NAME = 'target'\n        batch_size = 333  # Since the test data size is unknown, we batch it into the model.\nelse:    \n    class Config():\n        seed = 2024\n        num_folds = 10\n        TARGET_NAME = 'target'\n        batch_size = 1000  # Since the test data size is unknown, we batch it into the model.\n\nimport random  # Provides functions for generating random numbers.\n# Set the random seed to ensure model reproducibility.\ndef seed_everything(seed):\n    np.random.seed(seed)  # Set numpy's random seed.\n    random.seed(seed)  # Set Python's built-in random seed.\nseed_everything(Config.seed)\n\n# Read the data types of each feature in the training data\ncolname2dtype = pd.read_csv(\"/kaggle/input/home-credit-inconsistent-data-types/colname2dtype.csv\")\ncolname = colname2dtype['Column'].values\ndtype = colname2dtype['DataType'].values\n\ndtype2pl = {}\ndtype2pl['Int64'] = pl.Int64\ndtype2pl['Float64'] = pl.Float64\ndtype2pl['String'] = pl.String\ndtype2pl['Boolean'] = pl.String\n\ncolname2dtype = {}\nfor idx in range(len(colname)):\n    colname2dtype[colname[idx]] = dtype2pl[dtype[idx]]\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:15.074636Z","iopub.execute_input":"2024-12-09T21:37:15.075236Z","iopub.status.idle":"2024-12-09T21:37:15.104446Z","shell.execute_reply.started":"2024-12-09T21:37:15.075193Z","shell.execute_reply":"2024-12-09T21:37:15.103037Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find columns in the DataFrame 'df' where the proportion of missing values exceeds the specified 'margin', using pandas.\ndef find_df_null_col(df, margin=0.975):\n    cols = []\n    for col in df.columns:\n        if df[col].isna().mean() > margin:\n            cols.append(col)\n    return cols\n\n# For a file with multiple identical 'case_id' values, keep the last one.\n# Sometimes we need the most recent information for a specific user from certain files, and this function comes in handy.\ndef find_last_case_id(df, id='case_id'):  # Assuming the input 'df' is already sorted by 'case_id'.\n    df_copy = df.clone()\n    df_tail = df.tail(1)  # Extract the last 'case_id' separately.\n    # Find other 'case_id' values except the last one, shift is no longer needed, so drop it.\n    df_copy = df_copy.with_columns(pl.col(id).shift(-1).alias(f\"{id}_shift_-1\"))\n    df_last = df_copy.filter(pl.col(id) - pl.col(f'{id}_shift_-1') != 0).drop(f'{id}_shift_-1')\n    # Keep only the most recent information for each 'case_id'.\n    df_last = pl.concat([df_last, df_tail])\n    # Since this competition involves many files, it's essential to clean up memory promptly.\n    del df_copy, df_tail\n    gc.collect()  # Manually trigger garbage collection to force reclamation of memory marked as unused by the garbage collector.\n    return df_last\n\n# Fill the column 'col' in DataFrame 'df' using the specified 'method'.\ndef df_fillna(df, col, method=None):\n    if method is None:  # I don't intend to fill missing values in this column.\n        pass\n    elif method == \"forward\":  # Fill missing values using the previous value.\n        df = df.select([pl.col(col).fill_null('forward')])\n    else:  # For method=['NaN', 0], if treating missing values themselves as information, fill with \"NaN\".\n        df = df.with_columns(pl.col(col).fill_null(method).alias(col))\n    return df  # Return the DataFrame after filling missing values.\n\n# Perform one-hot encoding on the column 'col' in DataFrame 'df', providing the unique categories 'unique'.\ndef one_hot_encoder(df, col, unique):\n    # If there are only 2 categories, directly choose one of them.\n    if len(unique) == 2:\n        df = df.with_columns((pl.col(col) == unique[0]).cast(pl.Int8).alias(f\"{col}_{unique[0]}\"))\n    else:  # For multiple categories, handle them one by one.\n        for idx in range(len(unique)):\n            df = df.with_columns((pl.col(col) == unique[idx]).cast(pl.Int8).alias(f\"{col}_{unique[idx]}\"))\n    return df.drop(col)  # Drop the original column 'col' since it has been one-hot encoded.\n\n# Merge 'last_features', which contains the most recent information for each 'case_id', back into the original feature table 'feats'.\n# 'last_df' is the DataFrame with the most recent information for each 'case_id'.\ndef last_features_merge(feats, last_df, last_features=[]):\n    # Select the columns from 'last_df' that correspond to the features we want to summarize as most recent.\n    last_df = last_df.select(['case_id'] + [last[0] for last in last_features])\n    # Fill missing values in those columns of 'last_df'.\n    \n    # Fill missing values in the specified columns of 'last_df'.\n    for last in last_features:\n        col, fill = last\n        last_df = df_fillna(last_df, col, method=fill)\n\n    # After filling missing values, merge 'last_df' into the 'feats' table.\n    # The 'feats' columns still have missing values because some 'case_id' entries do not have data in 'last_df'.\n    feats = feats.join(last_df, on='case_id', how='left')\n    return feats\n\n# 'feats' contains the overall features, 'group_df' has multiple entries with the same 'case_id',\n# 'group_features' are the features to be used for grouping, and 'group_name' is the CSV file name.\n# Fill missing values, perform one-hot encoding, and group by.\ndef group_features_merge(feats, group_df, group_features=[], group_name='applprev2'):\n    # Select the 'group_features' columns from 'group_df'.\n    group_df = group_df.select(['case_id'] + [g[0] for g in group_features])\n    # Handle string columns separately.\n    for group in group_features:\n        if group_df[group[0]].dtype == pl.String:  # If it's a string type, perform one-hot encoding.\n            col, fill, one_hot = group\n            group_df = df_fillna(group_df, col, method=fill)  # Fill missing values as the first step.\n            if one_hot is None:  # If no one-hot encoding is required, drop the column.\n                group_df = group_df.drop(col)\n            else:  # Otherwise, perform one-hot encoding.\n                group_df = one_hot_encoder(group_df, col, one_hot)\n                for value in one_hot:\n                    new_col = f\"{col}_{value}\"\n                    feat = group_df.group_by('case_id').agg(\n                        pl.mean(new_col).alias(f\"mean_{group_name}_{new_col}\"),\n                        pl.std(new_col).alias(f\"std_{group_name}_{new_col}\"),\n                        pl.count(new_col).alias(f\"count_{group_name}_{new_col}\"),\n                    )\n                    feats = feats.join(feat, on='case_id', how='left')\n        else:  # If it's not a string (numeric) column, fill with the specified value.\n            col, fill = group\n            group_df = df_fillna(group_df, col, method=fill)  # Fill missing values as the first step.\n            feat = group_df.group_by('case_id').agg(\n                pl.max(col).alias(f\"max_{group_name}_{col}\"),\n                pl.mean(col).alias(f\"mean_{group_name}_{col}\"),\n                pl.median(col).alias(f\"median_{group_name}_{col}\"),\n                pl.std(col).alias(f\"std_{group_name}_{col}\"),\n                pl.min(col).alias(f\"min_{group_name}_{col}\"),\n                pl.count(col).alias(f\"count_{group_name}_{col}\"),\n                pl.sum(col).alias(f\"sum_{group_name}_{col}\"),\n                pl.n_unique(col).alias(f\"n_unique_{group_name}_{col}\"),\n                pl.first(col).alias(f\"first_{group_name}_{col}\"),\n                pl.last(col).alias(f\"last_{group_name}_{col}\")\n                                 )\n            feats=feats.join(feat,on='case_id',how='left')\n    return feats\n\ndef set_table_dtypes(df):\n    for col in df.columns:\n        df=df.with_columns(pl.col(col).cast(colname2dtype[col]).alias(col))\n    return df\n\n# \"after break\" refers to a detailed study of the meaning of each feature in the file.\ndef preprocessor(mode='train'):  # mode='train'|'test'\n    print(f\"{mode} base file after break. number 1\")\n    feats = pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_base.csv\").pipe(set_table_dtypes)\n    \n    if sample_run:\n        feats = feats.slice(0, 100)\n        \n    feats = feats.drop(['date_decision', 'MONTH', 'WEEK_NUM'])\n    print(\"-\" * 30)\n\n    print(f\"{mode} applprev_2 file after break. number: 1\")\n    applprev2 = pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_applprev_2.csv\").pipe(set_table_dtypes)\n    \n    if sample_run:\n        applprev2 = applprev2.slice(0, 100)  \n        \n    applprev2=applprev2.with_columns(\n                #The account has not been frozen, so there is no reason for freezing. No credit card application has been made before, and no contact information has been left.\n               ( (pl.col('cacccardblochreas_147M')!=pl.col('cacccardblochreas_147M'))&(pl.col('conts_type_509L')!=pl.col('conts_type_509L')) )\\\n                .alias(\"no_credit\")#.cast(pl.Int8)\n                )\n    applprev2=applprev2.with_columns(\n                #The account has not been frozen, so there is no reason for freezing, but a credit card application has been made.\n                ( (pl.col('cacccardblochreas_147M')!=pl.col('cacccardblochreas_147M'))&(pl.col('conts_type_509L')==pl.col('conts_type_509L'))) \\\n                .alias(\"no_frozen_credit\").cast(pl.Int8)\n                )\n    applprev2=applprev2.with_columns(\n                #There is a reason for freezing, so the account has been frozen, and naturally there is a credit card.\n                (pl.col('cacccardblochreas_147M')==pl.col('cacccardblochreas_147M'))\\\n                .alias(\"frozen_credit\").cast(pl.Int8)\n                )\n    \n    applprev2_last=find_last_case_id(applprev2)\n    \"\"\"\n    Some of these columns need to take the latest features, some need to be grouped.\n    The contact method needs to be the latest.\n    Check if a person's latest status still does not have a credit card.\n    Whether there is a credit card freeze also consider the latest status, anyway, just one feature.\n    The credit card freeze column feature can be constructed from the freeze reason column.\n    \"\"\"\n    # Here you only need to fill in the missing values and you can merge. The training data and test data strings will be one-hot together later.\n    last_features=[['conts_type_509L','WHATSAPP'],# There is only one WHATSAPP, so treat NaN as WHATSAPP.\n                   ['no_credit',0],\n                   ['no_frozen_credit',0],\n                   ['frozen_credit',0]\n                  ]\n    feats=last_features_merge(feats,applprev2_last,last_features)\n    \n    # groupby needs to consider fillna, onehot (if the string is None, don't one-hot, just drop it, if you want to one-hot, make a list of categories), then groupby, merge\n    group_features=[['cacccardblochreas_147M','a55475b1',\\\n                     [\"P19_60_110\",\"P17_56_144\",\"a55475b1\",\"P201_63_60\",\"P127_74_114\",\"P133_119_56\",\"P41_107_150\",\"P23_105_103\"\"P33_145_161\"]],\n                    ['credacc_cards_status_52L','UNCONFIRMED',\\\n                     ['BLOCKED','UNCONFIRMED','RENEWED', 'CANCELLED', 'INACTIVE', 'ACTIVE']],\n                     ['num_group1',0],#'num_group1', 'num_group2', temporarily not considered.\n                   ['num_group2',0],#'num_group1', 'num_group2', temporarily not considered.\n                   ]\n    feats=group_features_merge(feats,applprev2,group_features,group_name='applprev2')\n    del applprev2,applprev2_last\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    print(\"-\"*30)\n    \n    print(\"credit bureau b num 2\")\n    bureau_b_1=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_credit_bureau_b_1.csv\").pipe(set_table_dtypes)\n    bureau_b_2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_credit_bureau_b_2.csv\").pipe(set_table_dtypes)\n    bureau_b_1_last=find_last_case_id(bureau_b_1,id='case_id')\n    bureau_b_2_last=find_last_case_id(bureau_b_2,id='case_id')\n    feats=feats.join(bureau_b_1_last,on='case_id',how='left')\n    feats=feats.join(bureau_b_2_last,on='case_id',how='left')\n\n    del bureau_b_1,bureau_b_1_last,bureau_b_2,bureau_b_2_last\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n\n    print(f\"{mode} debitcard file after break num 1\")#'openingdate_857D': Debit card opening date. Temporarily not processed.\n    debitcard=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_debitcard_1.csv\").pipe(set_table_dtypes)\n    debitcard_last=find_last_case_id(debitcard,id='case_id')\n    \n    last_features=[['last180dayaveragebalance_704A',0],# Average balance of debit card within the past 180 days, fill with mode 0.\n                   ['last180dayturnover_1134A',30000],# Debit card turnover in the past 180 days, there is no particularly obvious mode here, fill with median.\n                   ['last30dayturnover_651A',0]# Fill with mode 0.\n                  ]\n    feats=last_features_merge(feats,debitcard_last,last_features)\n    group_features=[['num_group1',0]# Fill with mode.\n                  ]\n    feats=group_features_merge(feats,debitcard,group_features,group_name='debitcard')\n    del debitcard,debitcard_last\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    \n\n    print(f\"{mode} deposit file num 1\")\n    deposit=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_deposit_1.csv\").pipe(set_table_dtypes)\n    # Feature engineering of numerical columns. Start from 1 to remove 'case_id'.    \n    for idx in range(1,len(deposit.columns)):\n        col=deposit.columns[idx]\n        column_type = deposit[col].dtype\n        is_numeric = (column_type == pl.datatypes.Int64) or (column_type == pl.datatypes.Float64) \n        if is_numeric:# Construct features for numerical columns\n            feat=deposit.group_by('case_id').agg( pl.max(col).alias(f\"max_deposit_{col}\"),\n                                           pl.mean(col).alias(f\"mean_deposit_{col}\"),\n                                           pl.median(col).alias(f\"median_deposit_{col}\"),\n                                           pl.std(col).alias(f\"std_deposit_{col}\"),\n                                           pl.min(col).alias(f\"min_deposit_{col}\"),\n                                           pl.count(col).alias(f\"count_deposit_{col}\"),\n                                           pl.sum(col).alias(f\"sum_deposit_{col}\"),\n                                           pl.n_unique(col).alias(f\"n_unique_deposit_{col}\"),\n                                           pl.first(col).alias(f\"first_deposit_{col}\"),\n                                           pl.last(col).alias(f\"last_deposit_{col}\")\n                                         )\n            feats=feats.join(feat,on='case_id',how='left')\n    del deposit\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    \n    print(f\"{mode} other file after break number 1\")\n    other=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_other_1.csv\").pipe(set_table_dtypes)\n    other_last=find_last_case_id(other)\n    \n    # Here you only need to fill in the missing values and you can merge. The training data and test data strings will be one-hot together later.\n    last_features=[['amtdepositbalance_4809441A',0]# amtdepositbalance_4809441A: Customer deposit balance. Fill with mode 0.\n                  ]\n    feats=last_features_merge(feats,other_last,last_features)\n\n    group_features=[['amtdebitincoming_4809443A',0],# amtdebitincoming_4809443A, Incoming debit card transaction amount, fill with mode 0.\n                     ['amtdebitoutgoing_4809440A',0],# amtdebitoutgoing_4809440A, Outgoing debit card transaction amount, fill with mode 0.\n                     ['amtdepositincoming_4809444A',0], # amtdepositincoming_4809444A, Customer account deposit amount. The mode is 0.\n                     ['amtdepositoutgoing_4809442A',0]# amtdepositoutgoing_4809442A: Customer account withdrawal amount. The mode is 0.\n                   ]\n    feats=group_features_merge(feats,other,group_features,group_name='other')\n    \n    del other,other_last\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    \n    print(\"person 1 num 1\")\n    person1=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_person_1.csv\").pipe(set_table_dtypes)\n    # Directly drop columns with missing values >= 0.99.\n    person1=person1.drop(['birthdate_87D','childnum_185L','gender_992L','housingtype_772L','isreference_387L','maritalst_703L','role_993L'])                   \n    \n    person1=person1.select(['case_id','contaddr_matchlist_1032L','contaddr_smempladdr_334L','empl_employedtotal_800L','language1_981M',\n                           'persontype_1072L','persontype_792L','remitter_829L','role_1084L','safeguarantyflag_411L','sex_738L'])\n    person1_last=find_last_case_id(person1)\n    feats=feats.join(person1_last,on='case_id',how='left')\n    \n    del person1,person1_last\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    \n\n    print(f\"{mode} person2 file after break number 1\")\n    # After checking, the column dtypes of the person2 training set and test set correspond to each other.\n    person2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_person_2.csv\").pipe(set_table_dtypes)\n    # These features have a missing value ratio >= 0.96, no need to fill, just drop.\n    person2=person2.drop(['addres_role_871L','empls_employedfrom_796D','relatedpersons_role_762T'])\n    # Personal address, address postal code, employer name are considered private information, not used for training.\n    person2=person2.drop(['addres_district_368M','addres_zip_823M','empls_employer_name_740M'])\n    \n    group_features=[['conts_role_79M','a55475b1',# The contact role type of the person.\n                     ['a55475b1', 'P38_92_157', 'P7_147_157', 'P177_137_98', 'P125_14_176', \n                      'P125_105_50', 'P115_147_77', 'P58_79_51','P124_137_181', 'P206_38_166', 'P42_134_91']\n        ],\n                    ['empls_economicalst_849M','a55475b1',\n                    ['a55475b1', 'P164_110_33', 'P22_131_138', 'P28_32_178','P148_57_109', 'P7_47_145', 'P164_122_65', 'P112_86_147','P82_144_169', 'P191_80_124']\n                    ],\n                    ['num_group1',0],# Fill with mode 0.\n                   ['num_group2',0],# Fill with mode 0.\n                   ]\n    del person2\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    \n    print(f\"static_0 file num 2(3)\")\n    # pipe is used to customize your own functions on DataFrame\n    static_0_0=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_0_0.csv\").pipe(set_table_dtypes)\n    static_0_1=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_0_1.csv\").pipe(set_table_dtypes)\n    \n    static=pl.concat([static_0_0,static_0_1],how=\"vertical_relaxed\")# Vertical merge, and relaxed the restrictions on data type matching\n    if mode=='test':# If it is test data, there is another file\n        static_0_2=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_0_2.csv\").pipe(set_table_dtypes)\n        static=pl.concat([static,static_0_2],how=\"vertical_relaxed\")\n    feats=feats.join(static,on='case_id',how='left')\n    del static,static_0_0,static_0_1\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    \n    print(f\"{mode} static_cb_file after break num 1\")\n    static_cb=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_static_cb_0.csv\").pipe(set_table_dtypes)\n    # Directly drop columns with missing values >= 0.95.\n    static_cb=static_cb.drop(['assignmentdate_4955616D', 'dateofbirth_342D','for3years_128L',\n                            'for3years_504L','for3years_584L','formonth_118L','formonth_206L','formonth_535L',\n                           'forquarter_1017L', 'forquarter_462L','forquarter_634L','fortoday_1092L',\n                           'forweek_1077L','forweek_528L','forweek_601L','foryear_618L','foryear_818L','foryear_850L','pmtaverage_4955615A','pmtcount_4955617L','riskassesment_302T','riskassesment_940T'])\n    static_cb=static_cb.drop(['birthdate_574D','dateofbirth_337D',# Both are the customer's date of birth, temporarily not using this data.\n                             'assignmentdate_238D','assignmentdate_4527235D',# Tax authority data: assignment date and transfer date.\n                              'responsedate_1012D','responsedate_4527233D','responsedate_4917613D',# There are three features of the tax authority's reply date.\n                             ])\n# In static_cb, each case_id is a single data point, so we need to fill in missing values and then merge.\n    last_features=[ ['contractssum_5085716L',0],# The total contract value retrieved from external credit institutions\n                    ['days120_123L',0],# Number of credit bureau inquiries in the past 120 days, 0 is the mode but not prominent.\n                    ['days180_256L',0],# Number of credit bureau inquiries in the past 180 days, 0 is the mode but not prominent.\n                    ['days30_165L',0],# Number of credit bureau inquiries in the past 30 days, here 0 is a bit more prominent.\n                    ['days360_512L',1],# 1 is slightly more than 0.\n                    ['days90_310L',0],# 0 is a little more.\n                    ['description_5085714M','a55475b1'],# Classify customers according to the credit bureau. Binary classification of 10:1.\n                    #['education_1103M','a55475b1'],# The education level of the customer from an external source, 5 categories,\n                    ['education_88M','a55475b1'],# Customer's education level.\n                    ['firstquarter_103L',0],# Number of performances obtained from the credit bureau in the first quarter\n                    ['secondquarter_766L',0],# Number of performances in the second quarter.\n                    ['thirdquarter_1082L',0],# Number of performances in the third quarter.\n                    ['fourthquarter_440L',0],# Number of performances in the fourth quarter.\n                    ['maritalst_385M','a55475b1'],# Customer's marital status.\n                    #['maritalst_893M', 'a55475b1'],# Customer's marital status.\n                    ['numberofqueries_373L',1],# Number of inquiries to the credit bureau.\n                    ['pmtaverage_3A',0],# The average value of tax deductions\n                    #['pmtaverage_4527227A',7222.2],# The average value of tax deductions.\n                    #['pmtcount_4527229L', 6],# Number of tax deductions\n                    ['pmtcount_693L', 6],# Number of tax deductions\n                    ['pmtscount_423L',6.0],# Number of tax deduction payments.\n                    ['pmtssum_45A',0],# The total amount of tax deductions for the customer.\n                    ['requesttype_4525192L','DEDUCTION_6'],# Request type of the tax authority\n                  ]\n    feats=last_features_merge(feats,static_cb,last_features)\n    # Number of credit bureau inquiries in 60 days.\n    feats=feats.with_columns( (pl.col('days180_256L')-pl.col('days120_123L')).alias(\"daysgap60\"))\n    feats=feats.with_columns( (pl.col('days180_256L')-pl.col('days30_165L')).alias(\"daysgap150\"))\n    feats=feats.with_columns( (pl.col('days120_123L')-pl.col('days30_165L')).alias(\"daysgap90\"))\n    # Number of performances in a year.\n    feats=feats.with_columns( (pl.col('firstquarter_103L')+pl.col('secondquarter_766L')+pl.col('thirdquarter_1082L')+pl.col('fourthquarter_440L')).alias(\"totalyear_result\"))\n    \n    del static_cb\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    print(\"-\"*30)\n\n    print(f\"{mode} tax_a file after break num 1\")\n    tax_a=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_tax_registry_a_1.csv\").pipe(set_table_dtypes)\n    # The employer's name is private information, the data in the table is likely encrypted, so it's not useful. recorddate_4527225D is not used for now.\n    group_features=[['amount_4527230A',850],# The amount of tax deductions registered by the government, if there are missing values, fill with mode 850\n                     ['num_group1',0]\n                   ]\n    feats=group_features_merge(feats,tax_a,group_features,group_name='tax_a')\n    del tax_a\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    print(\"-\"*30)\n    \n    print(f\"{mode} tax_b file after break num 1\")\n    tax_b=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_tax_registry_b_1.csv\").pipe(set_table_dtypes)\n    # The employer's name is private information, cannot be used to train the model. num_group1, 'deductiondate_4917603D' is not used for now.\n    group_features=[['amount_4917619A',6885],# The amount of tax deductions tracked by the government registry, if there are missing values, fill with mode\n                    ['num_group1',0]\n                  ]\n    feats=group_features_merge(feats,tax_b,group_features,group_name='tax_b')\n    del tax_b\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    print(\"-\"*30)\n    \n    print(f\"{mode} tax_c file after break num 1\")\n    tax_c=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/{mode}/{mode}_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n    if len(tax_c)==0:\n        tax_c=pl.read_csv(f\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_tax_registry_c_1.csv\").pipe(set_table_dtypes)\n        \n    # employername_160M: The employer's name, not used because it's private information. processingdate_168D: The date of processing tax deductions.\n    tax_c=tax_c.drop(['employername_160M','processingdate_168D'])\n    \n    group_features=[['pmtamount_36A',850],# pmtamount_36A: The amount of tax deductions paid by the credit bureau, fill with mode 850\n                    ['num_group1',0]# 0 is the mode but it's not particularly prominent.\n                  ]\n    feats=group_features_merge(feats,tax_c,group_features,group_name='tax_c')\n    del tax_c\n    gc.collect()# Manually trigger garbage collection, forcibly recycle memory marked as unused by the garbage collector\n    print(\"-\"*30)\n    \n    return feats\ntrain_feats=preprocessor(mode='train')\ntest_feats=preprocessor(mode='test')\n\ntrain_feats=train_feats.to_pandas()\ntest_feats=test_feats.to_pandas()\n\n# Calculate the mode of each column, ignoring columns with missing values\nmode_values = train_feats.mode().iloc[0]\n# Use the mode to fill in missing values in the training set\ntrain_feats = train_feats.fillna(mode_values)\n# Use the mode to fill in missing values in the test set\ntest_feats = test_feats.fillna(mode_values)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:15.106617Z","iopub.execute_input":"2024-12-09T21:37:15.107107Z","iopub.status.idle":"2024-12-09T21:37:51.176886Z","shell.execute_reply.started":"2024-12-09T21:37:15.107065Z","shell.execute_reply":"2024-12-09T21:37:51.175552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Perform one-hot encoding transformation on string feature columns\nprint(\"----------string one hot encoder ****\")\nfor col in test_feats.columns:\n    n_unique=train_feats[col].nunique()\n    # If it's a categorical variable, perform one-hot encoding\n    # If the category is binary, like gender, if it's (0,1) or numeric type, there's no need to transform. If it's a string type, transform it into numeric\n    if n_unique==2 and train_feats[col].dtype=='object':\n        print(f\"one_hot_2:{col}\")\n        unique=train_feats[col].unique()\n        # Choose any category for transformation, for example, gender='Female'\n        train_feats[col]=(train_feats[col]==unique[0]).astype(int)\n        test_feats[col]=(test_feats[col]==unique[0]).astype(int)\n    elif (n_unique<10) and train_feats[col].dtype=='object': # Due to limited memory, the n_unique of categorical variables is set to 10\n        print(f\"one_hot_10:{col}\")\n        unique=train_feats[col].unique()\n        for idx in range(len(unique)):\n            if unique[idx]==unique[idx]: # This is to avoid the situation where there are nan values in the string\n                train_feats[col+\"_\"+str(idx)]=(train_feats[col]==unique[idx]).astype(int)\n                test_feats[col+\"_\"+str(idx)]=(test_feats[col]==unique[idx]).astype(int)\n        train_feats.drop([col],axis=1,inplace=True)\n        test_feats.drop([col],axis=1,inplace=True)\nprint(\"----------drop other string or unique value or full null value ****\")\ndrop_cols=[]\nfor col in test_feats.columns:\n    if (train_feats[col].dtype=='object') or (test_feats[col].dtype=='object') \\\n        or (train_feats[col].nunique()==1) or train_feats[col].isna().mean()>0.99:\n        drop_cols+=[col]\n# 'case_id' is not very useful.\ndrop_cols+=['case_id']\ntrain_feats.drop(drop_cols,axis=1,inplace=True)\ntest_feats.drop(drop_cols,axis=1,inplace=True)\nprint(f\"len(train_feats):{len(train_feats)},total_features_counts:{len(test_feats.columns)}\")\ntrain_feats.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:51.179148Z","iopub.execute_input":"2024-12-09T21:37:51.179514Z","iopub.status.idle":"2024-12-09T21:37:52.240622Z","shell.execute_reply.started":"2024-12-09T21:37:51.179485Z","shell.execute_reply":"2024-12-09T21:37:52.239508Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Traverse all columns of the dataframe df to change data types and reduce memory usage\ndef reduce_mem_usage(df, float16_as32=True):\n    # memory_usage() is the memory usage of each column of df, sum is their sum, B->KB->MB\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns: # Traverse each column name\n        col_type = df[col].dtype # Type of the column\n        if col_type != object: # Not an object, i.e., here we are dealing with numeric variables\n            c_min,c_max = df[col].min(),df[col].max() # Find the minimum and maximum of this column\n            if str(col_type)[:3] == 'int': # If it's an integer type variable, regardless of int8, int16, int32 or int64\n                # If the range of this column is within the range of int8, then convert the type (-128 to 127)\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                # If the range of this column is within the range of int16, then convert the type (-32,768 to 32,767)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                # If the range of this column is within the range of int32, then convert the type (-2,147,483,648 to 2,147,483,647)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                # If the range of this column is within the range of int64, then convert the type (-9,223,372,036,854,775,808 to 9,223,372,036,854,775,807)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else: # If it's a floating point type.\n                # If the value is within the range of float16, consider float32 if higher precision is needed\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32: # If higher precision is needed, choose float32\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float16)  \n                # If the value is within the range of float32, convert its type\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                # If the value is within the range of float64, convert its type\n                else:\n                    df[col] = df[col].astype(np.float64)\n    # Calculate the memory after the operation\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    # How much percentage has the memory decreased compared to the beginning\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\ntrain_feats = reduce_mem_usage(train_feats)\ntest_feats = reduce_mem_usage(test_feats)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.242941Z","iopub.execute_input":"2024-12-09T21:37:52.243407Z","iopub.status.idle":"2024-12-09T21:37:52.464632Z","shell.execute_reply.started":"2024-12-09T21:37:52.243367Z","shell.execute_reply":"2024-12-09T21:37:52.463344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_columns = train_feats.shape[1]\nprint(f\"The number of columns in train_feats is {num_columns}\")","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.465904Z","iopub.execute_input":"2024-12-09T21:37:52.466233Z","iopub.status.idle":"2024-12-09T21:37:52.472390Z","shell.execute_reply.started":"2024-12-09T21:37:52.466206Z","shell.execute_reply":"2024-12-09T21:37:52.470977Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Ran out of memory attempting to use loci for outlier detection.\n## Code in version 24","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.474000Z","iopub.execute_input":"2024-12-09T21:37:52.474493Z","iopub.status.idle":"2024-12-09T21:37:52.491403Z","shell.execute_reply.started":"2024-12-09T21:37:52.474454Z","shell.execute_reply":"2024-12-09T21:37:52.490201Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate the correlation of each feature with the target\ncorrelations = train_feats.corr()['target'].apply(abs).sort_values(ascending=False)\n\n# Get the top 20 most correlated features along with the target\nif sample_run:\n    top_corr_features = correlations.index[:5]\nelse:\n    top_corr_features = correlations.index[:20]\n\n# Create a new dataframe with the most correlated features\ntrain_feats_most_corr_20_target = train_feats[top_corr_features]\n\n# Print the correlations in a table\nprint(correlations.head(21))\n\n# Plot the correlations\nplt.figure(figsize=(12, 10))\nsns.heatmap(train_feats_most_corr_20_target.corr(), annot=True, cmap='coolwarm')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.493083Z","iopub.execute_input":"2024-12-09T21:37:52.493459Z","iopub.status.idle":"2024-12-09T21:37:52.913740Z","shell.execute_reply.started":"2024-12-09T21:37:52.493430Z","shell.execute_reply":"2024-12-09T21:37:52.912367Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# No feature is normally distributed\nHad to use sample here because running out of RAM. Repeated attempts supported no predictor was normally distributed.\n\nThis affects what methods can be used for anomaly detection.","metadata":{}},{"cell_type":"code","source":"# Initialize lists to hold the feature names\nnormal_features = []\nnot_normal_features = []\n\n# Define the sample size\nsample_size = min(100000, len(train_feats))\n\n# Iterate over the top 20 correlated features\nfor feature in top_corr_features:\n    # Get a sample of the data for the current feature\n    data_sample = train_feats[feature].sample(sample_size, random_state=0).dropna()\n\n    # Perform the D'Agostino's K^2^ test for normality\n    k2, p = normaltest(data_sample)\n\n    # If p < 0.05, the null hypothesis is rejected and the distribution is not normal\n    if p < 0.05:\n        not_normal_features.append(feature)\n    else:\n        normal_features.append(feature)\n\n# Print the results\nprint(\"Normal features:\", normal_features)\nprint(\"Not normal features:\", not_normal_features)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.915254Z","iopub.execute_input":"2024-12-09T21:37:52.915724Z","iopub.status.idle":"2024-12-09T21:37:52.947389Z","shell.execute_reply.started":"2024-12-09T21:37:52.915683Z","shell.execute_reply":"2024-12-09T21:37:52.946087Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Most correlated predictor\nThe most correlated predictor is the percentage of payments 4 or more days past poor date\npctinstlsallpaidlate4d_3546849L\n\nHowever, the correlation is weak.","metadata":{}},{"cell_type":"code","source":"# Remember that absolute values of correlations are above\nprint(train_feats['target'].corr(train_feats['pctinstlsallpaidlate4d_3546849L']))","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.952278Z","iopub.execute_input":"2024-12-09T21:37:52.952675Z","iopub.status.idle":"2024-12-09T21:37:52.968115Z","shell.execute_reply.started":"2024-12-09T21:37:52.952647Z","shell.execute_reply":"2024-12-09T21:37:52.966974Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the top 10 most correlated features along with the target\ntop_10_corr_features = correlations.index[:11]\n\n# Create a new dataframe with the most correlated features\ntrain_feats_most_corr_10_target = train_feats[top_10_corr_features]\n\n# Plot the correlations\nplt.figure(figsize=(12, 10))\nsns.heatmap(train_feats_most_corr_10_target.corr(), annot=True, cmap='coolwarm')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:52.969504Z","iopub.execute_input":"2024-12-09T21:37:52.969913Z","iopub.status.idle":"2024-12-09T21:37:53.653960Z","shell.execute_reply.started":"2024-12-09T21:37:52.969878Z","shell.execute_reply":"2024-12-09T21:37:53.652709Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_corr_features","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:53.655865Z","iopub.execute_input":"2024-12-09T21:37:53.656198Z","iopub.status.idle":"2024-12-09T21:37:53.663151Z","shell.execute_reply.started":"2024-12-09T21:37:53.656172Z","shell.execute_reply":"2024-12-09T21:37:53.661874Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Extreme quantile-based outlier elimination\nKNN below may be used later. Would need to use sampling.\nThe target = 1 case is <3% of records as I call so only looking for quantile outliers where target = 0.","metadata":{}},{"cell_type":"code","source":"# Calculate the quantiles (1% and 99%)\ntrain_feats['base_row_number'] = range(len(train_feats))\ntrain_feats['base_row_number'] = train_feats['base_row_number'] + 1\ntarget0_df = train_feats[train_feats['target'] == 0]\nq_low = target0_df.quantile(0.001)\nq_high = target0_df.quantile(0.999)\n\n# For finding lower and upper outliers\nlow_iter  = 0\nhigh_iter = 0\n# Create a mask for each column\ncolumn_masks = {}\nrow_numbers_outliers_target0 = []\nunique_outlier_cols = []\n\ncolumns_to_exclude = ['base_number']\n\nfor col in train_feats.columns:\n    if col in q_low and col in q_high:\n        column_masks[col] = (target0_df[col] < q_low[col]) | (target0_df[col] > q_high[col])\n        if column_masks[col].any():\n            unique_outlier_cols.append(col)\nunique_outlier_cols = list(set(unique_outlier_cols))\n\nfor row_index, row in target0_df.iterrows():\n    # Check if any column value exceeds the 10th quartile\n    condition = pd.Series([False] * len(row), index=row.index)\n    for col in row.index.difference(columns_to_exclude):\n        condition[col] = (row[col] < q_low[col]) | (row[col] > q_high[col])\n    if condition.any():\n        row_numbers_outliers_target0.append(row['base_row_number'])\n        for col in row.index.difference(columns_to_exclude):\n            if row[col] < q_low[col]:\n                    low_iter+=1\n                    if low_iter < 3:                        \n                        print(f\"Example lower outlier for column {col}: {row[col]}, q_low: {q_low[col]}\")\n                        break\n            if row[col] > q_high[col]:\n                    high_iter+=1\n                    if high_iter < 3:                        \n                        print(f\"Example upper outlier for column {col}: {row[col]}, q_high: {q_high[col]}\")\n                        break\n\nmask = train_feats['base_row_number'].isin(row_numbers_outliers_target0)\ntrain_feats.drop('base_row_number', axis=1, inplace=True)\n\neliminated_records = train_feats.loc[mask]\n# Drop the rows based on the mask\ntrain_feats.drop(train_feats[mask].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:53.664783Z","iopub.execute_input":"2024-12-09T21:37:53.665366Z","iopub.status.idle":"2024-12-09T21:37:54.322656Z","shell.execute_reply.started":"2024-12-09T21:37:53.665321Z","shell.execute_reply":"2024-12-09T21:37:54.321471Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Target distribution for outliers","metadata":{}},{"cell_type":"code","source":"relative_frequency_target_outliers = eliminated_records['target'].value_counts(normalize=True)\nrelative_frequency_target = train_feats['target'].value_counts(normalize=True)\nprint(relative_frequency_target_outliers)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.323933Z","iopub.execute_input":"2024-12-09T21:37:54.324252Z","iopub.status.idle":"2024-12-09T21:37:54.332445Z","shell.execute_reply.started":"2024-12-09T21:37:54.324224Z","shell.execute_reply":"2024-12-09T21:37:54.331314Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Target distribution for all data","metadata":{}},{"cell_type":"code","source":"print(relative_frequency_target)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.333678Z","iopub.execute_input":"2024-12-09T21:37:54.333985Z","iopub.status.idle":"2024-12-09T21:37:54.346291Z","shell.execute_reply.started":"2024-12-09T21:37:54.333961Z","shell.execute_reply":"2024-12-09T21:37:54.345075Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Unique columns in the quantile outlier test","metadata":{}},{"cell_type":"code","source":"unique_values_outliers = sorted(unique_outlier_cols)\n\n# Get the first 10 characters of each unique value\nfirst_10_chars = [val[:10] for val in train_feats.columns]\nfirst_10_chars_outliers = [val[:10] for val in unique_values_outliers]\n\n# Count the frequencies of each 10-character string\ncounter = Counter(first_10_chars)\nall_columns_first_10_chars = counter.most_common()\n\ncounter_outliers = Counter(first_10_chars_outliers)\n\nrank_dict = {k: rank+1 for rank, (k, v) in enumerate(all_columns_first_10_chars)}\n\n\n# Calculate total number of records\ntotal_records = len(first_10_chars)\ntotal_records_outliers = len(first_10_chars_outliers)\n\n# Get the most common 10-character strings, sorted by frequency\n# with percentage of those column prefixes found in the outliers df then entire df\nmost_common_outliers = [(k, v, v/total_records_outliers*100, rank_dict.get(k, 'N/A'), i+1) for i, (k, v) in enumerate(counter_outliers.most_common(20))]\n\ndf_most_common_outliers = pd.DataFrame(most_common_outliers, columns=['Col_Pre', 'Outlier_Freq', 'Outlier_Col_Pct', 'All_Data_Rank', 'Outlier_Rank'])\n\npprint.pprint(df_most_common_outliers)\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.347536Z","iopub.execute_input":"2024-12-09T21:37:54.347860Z","iopub.status.idle":"2024-12-09T21:37:54.363447Z","shell.execute_reply.started":"2024-12-09T21:37:54.347833Z","shell.execute_reply":"2024-12-09T21:37:54.362329Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# knn_outlier_test not completing, will change this code to a more interesting test than Z-score outlier\n\n# This line would require internet to be on and installing pyod\n# \"Internet on\" eliminates ability to submit for this competition\n\n# from pyod.models.knn import KNN\n# \n# import numpy as np\n# import pandas as pd\n# from pyod.models.knn import KNN\n# \n# def knn_outlier_test(df):\n#     results = []\n#     for column in df.select_dtypes(include=[np.number]).columns:\n#         data = df[column].values.reshape(-1, 1)\n#         # n_jobs=-1 for parallel processing\n#         clf = KNN(contamination=0.02, n_jobs=-1)  # initialize detector\n#         clf.fit(data)  # fit the model\n#         outliers = clf.predict(data)  # get the prediction labels of the training data\n#         outlier_indexes = np.where(outliers == 1)[0]  # get the outlier indices\n#         \n#         for outlier_index in outlier_indexes:\n#             results.append((column, 'KNN', outlier_index, data[outlier_index][0]))\n#     \n#     return pd.DataFrame(results, columns=['Column', 'Test Used', 'Observation', 'Outlier Value'])\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.365194Z","iopub.execute_input":"2024-12-09T21:37:54.365650Z","iopub.status.idle":"2024-12-09T21:37:54.381256Z","shell.execute_reply.started":"2024-12-09T21:37:54.365613Z","shell.execute_reply":"2024-12-09T21:37:54.380177Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Key parameter to adjust : Pearson Correlation for Linear Regression\n\nAdjust number in this line below to easily use different variables. Experiment to see how this affects the score when you submit.\n\n> if abs(pearson)>0.0020:","metadata":{}},{"cell_type":"code","source":"def pearson_corr(x1,x2):\n    \"\"\"\n    x1,x2:np.array\n    \"\"\"\n    mean_x1=np.mean(x1)\n    mean_x2=np.mean(x2)\n    std_x1=np.std(x1)\n    std_x2=np.std(x2)\n    pearson=np.mean((x1-mean_x1)*(x2-mean_x2))/(std_x1*std_x2)\n    return pearson\n\n# Are there any features that are particularly highly correlated with the target? Use them for logistic regression.\nchoose_cols=[]\nall_correlations = {}  # Create a dictionary to store all correlations\n\nfor col in train_feats.columns:\n    if col != 'target':\n        pearson=pearson_corr(train_feats[col].values,train_feats['target'].values) \n        all_correlations[col] = pearson  # Store the correlation in the dictionary\n        # .0025 yielded score of 0.5 in early version of notebook\n        if abs(pearson)>0.0024:\n            choose_cols.append(col)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.382639Z","iopub.execute_input":"2024-12-09T21:37:54.383093Z","iopub.status.idle":"2024-12-09T21:37:54.433815Z","shell.execute_reply.started":"2024-12-09T21:37:54.383055Z","shell.execute_reply":"2024-12-09T21:37:54.432568Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"type(correlations)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.435403Z","iopub.execute_input":"2024-12-09T21:37:54.435877Z","iopub.status.idle":"2024-12-09T21:37:54.444360Z","shell.execute_reply.started":"2024-12-09T21:37:54.435825Z","shell.execute_reply":"2024-12-09T21:37:54.443168Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Drop predictors - multicolinearity based on VIF","metadata":{}},{"cell_type":"code","source":"#mean_gini:0.5428968427934477 with abs(pearson)>0.0025? Might check\n#mean_gini:0.5415694487332248 with abs(pearson)>0.0024\nfrom sklearn.linear_model import LinearRegression\n\nX=train_feats[choose_cols].copy()\ny=train_feats[Config.TARGET_NAME].copy()\ntest_X=test_feats[choose_cols].copy()\noof_pred_pro=np.zeros((len(X)))\ntest_pred_pro=np.zeros((Config.num_folds,len(test_X)))\ndel train_feats,test_feats\ngc.collect() # This line manually triggers garbage collection to forcibly reclaim memory that the garbage collector has marked as unused.\n\nskf = StratifiedKFold(n_splits=Config.num_folds,random_state=Config.seed, shuffle=True)\n\nfor fold, (train_index, valid_index) in (enumerate(skf.split(X, y.astype(str)))):\n    print(f\"fold:{fold}\")\n\n    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n\n    # “create a linear regression model”.\n    model = LinearRegression()\n    model.fit(X_train,y_train)\n\n    oof_pred_pro[valid_index]=model.predict(X_valid)\n    # “predict the data in batches”.\n    for idx in range(0,len(test_X),Config.batch_size):\n        test_pred_pro[fold][idx:idx+Config.batch_size]=model.predict(test_X[idx:idx+Config.batch_size]) \n    del model,X_train, X_valid,y_train, y_valid #This line deletes the model and the training and validation sets after they have been used.\n    gc.collect() #This line manually triggers garbage collection to forcibly reclaim memory that the garbage collector has marked as unused.\ngini=2*roc_auc_score(y.values,oof_pred_pro)-1\nprint(f\"mean_gini:{gini}\")","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.445694Z","iopub.execute_input":"2024-12-09T21:37:54.446219Z","iopub.status.idle":"2024-12-09T21:37:54.930810Z","shell.execute_reply.started":"2024-12-09T21:37:54.446178Z","shell.execute_reply":"2024-12-09T21:37:54.929631Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sampling because VIF ran out of memory\ndef calculate_vif_drop_common_columns(X, num_iterations=3, thresh=10, seed=39):\n    common_columns = set()  # Initialize an empty set\n\n    for _ in range(num_iterations):   \n        print(\"Iteration \" + str(_))     \n        sampled_df = X.sample(frac=0.2, random_state=seed)\n\n        # Calculate VIF for each column\n        vif = [variance_inflation_factor(sampled_df.values, ix)\n               for ix in range(sampled_df.shape[1])]\n        \n        columns_to_drop = [sampled_df.columns[ix] for ix, vif_value in enumerate(vif) if vif_value > thresh]\n\n        # This causes the first iteration to provide the columns that must then appear in all other iterations for it to be dropped.\n        if not common_columns:\n            common_columns.update(columns_to_drop)\n        else:\n            common_columns.intersection_update(columns_to_drop)\n\n    # Drop columns that appear in each iteration of VIF\n    X_dropped = X.drop(columns=common_columns)\n\n    return X_dropped","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.932041Z","iopub.execute_input":"2024-12-09T21:37:54.932407Z","iopub.status.idle":"2024-12-09T21:37:54.940201Z","shell.execute_reply.started":"2024-12-09T21:37:54.932379Z","shell.execute_reply":"2024-12-09T21:37:54.938964Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = calculate_vif_drop_common_columns(X)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:54.941471Z","iopub.execute_input":"2024-12-09T21:37:54.941909Z","iopub.status.idle":"2024-12-09T21:37:55.143964Z","shell.execute_reply.started":"2024-12-09T21:37:54.941869Z","shell.execute_reply":"2024-12-09T21:37:55.142807Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Questionable Residual Plot - binary DV\nThis should be interpreted cautiously given the binary dependent variable and multiple predictors.\n\nInteresting outliers in the plot are for the very high and very low predicted values.","metadata":{}},{"cell_type":"markdown","source":"## Residual Outlier Analysis","metadata":{}},{"cell_type":"markdown","source":"Interesting outliers in the plot are for the very high and very low predicted values.\n\nWhy are these predicted at such extremes, which happen to be wrong?","metadata":{}},{"cell_type":"code","source":"full_model = LinearRegression()\nfull_model.fit(X,y)\n\ny_pred = full_model.predict(X)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.145383Z","iopub.execute_input":"2024-12-09T21:37:55.145672Z","iopub.status.idle":"2024-12-09T21:37:55.157876Z","shell.execute_reply.started":"2024-12-09T21:37:55.145650Z","shell.execute_reply":"2024-12-09T21:37:55.156569Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Calculate the residuals\nresiduals = y - y_pred\n\n# Create a scatter plot of predicted vs residuals\nplt.figure(figsize=(8, 6))\nsns.scatterplot(x=y_pred, y=residuals)\nplt.xlabel('Predicted')\nplt.ylabel('Residuals')\nplt.axhline(y=0, color='r', linestyle='--')\nplt.title('Residuals vs Predicted')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.159394Z","iopub.execute_input":"2024-12-09T21:37:55.159832Z","iopub.status.idle":"2024-12-09T21:37:55.436162Z","shell.execute_reply.started":"2024-12-09T21:37:55.159781Z","shell.execute_reply":"2024-12-09T21:37:55.435043Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Clipping Predictions","metadata":{}},{"cell_type":"code","source":"# Use numpy's clip function to limit values\ny_pred = np.clip(y_pred, 0, 1)\ngini = 2 * roc_auc_score(y, y_pred) - 1\nprint(f\"mean_gini:{gini}\")","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.437483Z","iopub.execute_input":"2024-12-09T21:37:55.437810Z","iopub.status.idle":"2024-12-09T21:37:55.447472Z","shell.execute_reply.started":"2024-12-09T21:37:55.437783Z","shell.execute_reply":"2024-12-09T21:37:55.446298Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds=test_pred_pro.mean(axis=0)\ntest_preds=np.clip(test_preds, 0, 1)\nsubmission=pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\n# Replaces NaN with 0.3 and makes values <0 = 0 and >1 = 1\nsubmission['score']=np.clip(np.nan_to_num(test_preds,nan=0.3),0,1)\nsubmission.to_csv(\"submission.csv\",index=None)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.448932Z","iopub.execute_input":"2024-12-09T21:37:55.449447Z","iopub.status.idle":"2024-12-09T21:37:55.474012Z","shell.execute_reply.started":"2024-12-09T21:37:55.449417Z","shell.execute_reply":"2024-12-09T21:37:55.472801Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Evaluating relative frequency of target : submission vs. training data","metadata":{}},{"cell_type":"code","source":"relative_frequency_submission = submission['score'].value_counts(normalize=True)\nrelative_frequency_submission_rounded = round(submission['score']).value_counts(normalize=True)\n## Defined above, but train_feats dropped\nrelative_frequency_target = y.value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.475440Z","iopub.execute_input":"2024-12-09T21:37:55.475768Z","iopub.status.idle":"2024-12-09T21:37:55.484355Z","shell.execute_reply.started":"2024-12-09T21:37:55.475740Z","shell.execute_reply":"2024-12-09T21:37:55.483065Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create dot plot\nplt.figure(figsize=(10, 5))\nplt.plot(relative_frequency_submission, 'o')\n\n# Set labels\nplt.xlabel('Score')\nplt.ylabel('Relative Frequency')\nplt.title('Dot Plot of Relative Frequency of Scores')\n\n# Show the plot\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.488919Z","iopub.execute_input":"2024-12-09T21:37:55.489317Z","iopub.status.idle":"2024-12-09T21:37:55.768577Z","shell.execute_reply.started":"2024-12-09T21:37:55.489275Z","shell.execute_reply":"2024-12-09T21:37:55.767293Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(relative_frequency_submission_rounded)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.770148Z","iopub.execute_input":"2024-12-09T21:37:55.771188Z","iopub.status.idle":"2024-12-09T21:37:55.777309Z","shell.execute_reply.started":"2024-12-09T21:37:55.771146Z","shell.execute_reply":"2024-12-09T21:37:55.775989Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(relative_frequency_target)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T21:37:55.778694Z","iopub.execute_input":"2024-12-09T21:37:55.779019Z","iopub.status.idle":"2024-12-09T21:37:55.793229Z","shell.execute_reply.started":"2024-12-09T21:37:55.778993Z","shell.execute_reply":"2024-12-09T21:37:55.792078Z"},"trusted":true},"outputs":[],"execution_count":null}]}