{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n# import dask.dataframe as dd","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:02:08.504057Z","iopub.execute_input":"2024-04-10T23:02:08.505061Z","iopub.status.idle":"2024-04-10T23:02:12.626547Z","shell.execute_reply.started":"2024-04-10T23:02:08.505017Z","shell.execute_reply":"2024-04-10T23:02:12.625448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Selection Step","metadata":{"tags":[]}},{"cell_type":"code","source":"train_labels = pl.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:02:14.804520Z","iopub.execute_input":"2024-04-10T23:02:14.805185Z","iopub.status.idle":"2024-04-10T23:02:15.142090Z","shell.execute_reply.started":"2024-04-10T23:02:14.805150Z","shell.execute_reply":"2024-04-10T23:02:15.140818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels.head()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-08T21:48:57.744277Z","iopub.execute_input":"2024-04-08T21:48:57.745392Z","iopub.status.idle":"2024-04-08T21:48:57.749111Z","shell.execute_reply.started":"2024-04-08T21:48:57.745353Z","shell.execute_reply":"2024-04-08T21:48:57.748043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = train_labels.sample((len(train_labels)*0.2), seed = 16  )","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:02:17.895741Z","iopub.execute_input":"2024-04-10T23:02:17.896152Z","iopub.status.idle":"2024-04-10T23:02:17.974742Z","shell.execute_reply.started":"2024-04-10T23:02:17.896118Z","shell.execute_reply":"2024-04-10T23:02:17.973468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:02:20.070860Z","iopub.execute_input":"2024-04-10T23:02:20.071250Z","iopub.status.idle":"2024-04-10T23:02:20.079034Z","shell.execute_reply.started":"2024-04-10T23:02:20.071221Z","shell.execute_reply":"2024-04-10T23:02:20.078041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_labels.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:02:22.599323Z","iopub.execute_input":"2024-04-10T23:02:22.600579Z","iopub.status.idle":"2024-04-10T23:02:22.614169Z","shell.execute_reply.started":"2024-04-10T23:02:22.600529Z","shell.execute_reply":"2024-04-10T23:02:22.612613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pl.read_csv('/kaggle/input/amex-default-prediction/train_data.csv')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:02:30.118226Z","iopub.execute_input":"2024-04-10T23:02:30.118604Z","iopub.status.idle":"2024-04-10T23:04:06.768617Z","shell.execute_reply.started":"2024-04-10T23:02:30.118577Z","shell.execute_reply":"2024-04-10T23:04:06.767087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:04:27.954663Z","iopub.execute_input":"2024-04-10T23:04:27.955384Z","iopub.status.idle":"2024-04-10T23:04:27.962947Z","shell.execute_reply.started":"2024-04-10T23:04:27.955349Z","shell.execute_reply":"2024-04-10T23:04:27.961720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"development_sample = train_labels.join(train_data, on='customer_ID', how='inner')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:04:29.113517Z","iopub.execute_input":"2024-04-10T23:04:29.113971Z","iopub.status.idle":"2024-04-10T23:04:32.284084Z","shell.execute_reply.started":"2024-04-10T23:04:29.113912Z","shell.execute_reply":"2024-04-10T23:04:32.283185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploration","metadata":{}},{"cell_type":"code","source":"development_sample.head()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:04:41.722159Z","iopub.execute_input":"2024-04-10T23:04:41.722556Z","iopub.status.idle":"2024-04-10T23:04:41.745246Z","shell.execute_reply.started":"2024-04-10T23:04:41.722518Z","shell.execute_reply":"2024-04-10T23:04:41.744194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"development_sample.describe()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-09T05:23:33.970351Z","iopub.execute_input":"2024-04-09T05:23:33.970790Z","iopub.status.idle":"2024-04-09T05:23:45.282338Z","shell.execute_reply.started":"2024-04-09T05:23:33.970757Z","shell.execute_reply":"2024-04-09T05:23:45.281021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"development_sample = development_sample.with_columns(\n    pl.col('S_2').str.strptime(pl.Date, '%Y-%m-%d')\n)\n","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:04:54.309763Z","iopub.execute_input":"2024-04-10T23:04:54.310816Z","iopub.status.idle":"2024-04-10T23:04:54.360271Z","shell.execute_reply.started":"2024-04-10T23:04:54.310777Z","shell.execute_reply":"2024-04-10T23:04:54.359018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"development_sample.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:03.847121Z","iopub.execute_input":"2024-04-10T23:05:03.847497Z","iopub.status.idle":"2024-04-10T23:05:03.856005Z","shell.execute_reply.started":"2024-04-10T23:05:03.847469Z","shell.execute_reply":"2024-04-10T23:05:03.854496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"development_sample = development_sample.drop('months_of_data_right')","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:06.463185Z","iopub.execute_input":"2024-04-10T23:05:06.463564Z","iopub.status.idle":"2024-04-10T23:05:06.468999Z","shell.execute_reply.started":"2024-04-10T23:05:06.463528Z","shell.execute_reply":"2024-04-10T23:05:06.467695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of occurrences for each customer ID\ncustomer_month_counts = development_sample.groupby('customer_ID').agg(\n    months_of_data=pl.count()\n)\n# Merge this count back with the original dataframe to get the target variable for each customer\ndevelopment_sample = development_sample.join(customer_month_counts, on='customer_ID')","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:16.122788Z","iopub.execute_input":"2024-04-10T23:05:16.124048Z","iopub.status.idle":"2024-04-10T23:05:17.647810Z","shell.execute_reply.started":"2024-04-10T23:05:16.124002Z","shell.execute_reply":"2024-04-10T23:05:17.646806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(development_sample.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-10T16:37:54.286925Z","iopub.execute_input":"2024-04-10T16:37:54.287277Z","iopub.status.idle":"2024-04-10T16:37:54.302738Z","shell.execute_reply.started":"2024-04-10T16:37:54.287249Z","shell.execute_reply":"2024-04-10T16:37:54.301540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group by the number of months of data and calculate the number of unique customer IDs and default rate\nsummary_table = development_sample.groupby('months_of_data').agg(\n    Number_of_Observations=pl.col('customer_ID').n_unique(),\n    Default_Rate=pl.col('target').mean()\n).sort('months_of_data')\n\n# Convert the summary table to a pandas DataFrame for display\nsummary_table_df = summary_table.to_pandas()\n\nsummary_table_df = summary_table_df.sort_values(by='months_of_data', ascending=False)\n\n# Display the summary table\nprint(summary_table_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:25.809359Z","iopub.execute_input":"2024-04-10T23:05:25.809768Z","iopub.status.idle":"2024-04-10T23:05:26.203895Z","shell.execute_reply.started":"2024-04-10T23:05:25.809738Z","shell.execute_reply":"2024-04-10T23:05:26.202644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of features in each category\ncategories = {\n    'D_': 'Delinquency variables',\n    'S_': 'Spend variables',\n    'P_': 'Payment variables',\n    'B_': 'Balance variables',\n    'R_': 'Risk variables'\n}\n\nfeature_counts = []\nfor prefix, category in categories.items():\n    count = sum(feature.startswith(prefix) for feature in development_sample.columns)\n    feature_counts.append({'Category': category, '# of features': count})\n\n# Convert the list to a Pandas DataFrame\nfeature_counts_df = pd.DataFrame(feature_counts)\n\n# Display the table\nprint(feature_counts_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:36.286953Z","iopub.execute_input":"2024-04-10T23:05:36.287966Z","iopub.status.idle":"2024-04-10T23:05:36.299993Z","shell.execute_reply.started":"2024-04-10T23:05:36.287900Z","shell.execute_reply":"2024-04-10T23:05:36.298243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_columns = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\nall_columns = [c for c in list(development_sample.columns) if c not in ['customer_ID','S_2']]\n\ncontinuous_columns = [col for col in all_columns if col not in categorical_columns + ['target']]","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-04-10T23:05:39.412083Z","iopub.execute_input":"2024-04-10T23:05:39.412498Z","iopub.status.idle":"2024-04-10T23:05:39.420113Z","shell.execute_reply.started":"2024-04-10T23:05:39.412466Z","shell.execute_reply":"2024-04-10T23:05:39.418437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in continuous_columns:\n    development_sample = development_sample.cast({col: pl.Float64})","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:42.880606Z","iopub.execute_input":"2024-04-10T23:05:42.881009Z","iopub.status.idle":"2024-04-10T23:05:43.347993Z","shell.execute_reply.started":"2024-04-10T23:05:42.880974Z","shell.execute_reply":"2024-04-10T23:05:43.346565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the reference date (most recent date)\nreference_date = pl.datetime(2018, 4, 30)\n\n# # # Define a helper function to create aggregated features\n# def create_aggregated_features(df, feature, months, agg_func, agg_func_name, reference_date):\n#     # Filter the DataFrame to include only the last 'months' months\n#     filtered_df = df.filter(pl.col('S_2') >= (reference_date - pl.duration(days = 30*months)))\n    \n#     # Group by 'customer_ID' and aggregate the feature\n#     agg_df = filtered_df.group_by('customer_ID').agg([\n#         agg_func(feature).alias(f\"{feature}_{agg_func_name}_{months}\")\n#     ])\n    \n#     return agg_df\n\n# # Initialize a list to store the feature DataFrames\n# feature_dfs = []\n\n# # Iterate over the continuous columns and create aggregated features\n# for feature in continuous_columns:\n#     for months, agg_func, agg_func_name in [(6, pl.mean, 'mean'), (12, pl.mean, 'mean'), \n#                                              (6, pl.min, 'min'), (9, pl.min, 'min'), \n#                                              (3, pl.sum, 'sum')]:\n#         feature_dfs.append(create_aggregated_features(development_sample, feature, months, agg_func, agg_func_name, reference_date))\n\n\n\n# -------------------------------------------------------------------------------\n\n\n\n# Define a function to create aggregated features\ndef create_aggregated_features(df, feature, months, reference_date):\n    # Filter the DataFrame to include only the last 'months' months\n    filtered_df = df.filter(pl.col('S_2') >= (reference_date - pl.duration(days=30*months)))\n    \n    # Group by 'customer_ID' and calculate aggregated features\n    agg_df = filtered_df.groupby('customer_ID').agg([\n        pl.col(feature).mean().alias(f\"{feature}_Ave_{months}\"),\n        pl.col(feature).min().alias(f\"{feature}_Min_{months}\"),\n        pl.col(feature).max().alias(f\"{feature}_Max_{months}\")\n    ])\n    \n    return agg_df\n\n# Define the list of numerical features for which you want to create aggregated features\nnumerical_features = continuous_columns  # Assuming 'continuous_columns' contains the list of numerical features\n\n# Initialize an empty list to store the feature DataFrames\nfeature_dfs = []","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:05:52.735744Z","iopub.execute_input":"2024-04-10T23:05:52.736196Z","iopub.status.idle":"2024-04-10T23:05:52.745395Z","shell.execute_reply.started":"2024-04-10T23:05:52.736163Z","shell.execute_reply":"2024-04-10T23:05:52.744138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize an empty list to store the feature DataFrames\nfeature_dfs = []\n\n# Iterate over the numerical features and create aggregated features for different time periods\nfor feature in continuous_columns:\n    for months in [6,12]:  # Specify the time periods you want to aggregate over\n        feature_df = create_aggregated_features(development_sample, feature, months, reference_date)\n        # Keep 'customer_ID' only in the first DataFrame\n        if feature_dfs:\n            feature_df = feature_df.drop('customer_ID')\n        feature_dfs.append(feature_df)\n\n# Combine all aggregated feature DataFrames into a single DataFrame\nfinal_features_df = pl.concat(feature_dfs, how='horizontal')\n\n# Add 'customer_ID' back to the final DataFrame by joining with the first DataFrame in feature_dfs\nfinal_features_df = feature_dfs[0].select('customer_ID').join(final_features_df, on='customer_ID', how='inner')\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:06:03.573291Z","iopub.execute_input":"2024-04-10T23:06:03.573673Z","iopub.status.idle":"2024-04-10T23:08:58.443926Z","shell.execute_reply.started":"2024-04-10T23:06:03.573645Z","shell.execute_reply":"2024-04-10T23:08:58.442989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cat = development_sample[categorical_columns + ['customer_ID', 'S_2']]\n# for col in categorical_columns:\n#     df_cat = df_cat.cast({col: pl.Float64})","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:09:02.728081Z","iopub.execute_input":"2024-04-10T23:09:02.728491Z","iopub.status.idle":"2024-04-10T23:09:02.735238Z","shell.execute_reply.started":"2024-04-10T23:09:02.728460Z","shell.execute_reply":"2024-04-10T23:09:02.733786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cat = df_cat.to_dummies(columns = categorical_columns)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:09:05.894294Z","iopub.execute_input":"2024-04-10T23:09:05.894708Z","iopub.status.idle":"2024-04-10T23:09:06.159586Z","shell.execute_reply.started":"2024-04-10T23:09:05.894675Z","shell.execute_reply":"2024-04-10T23:09:06.158630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a helper function to create aggregated features for binary categorical features\ndef create_categorical_aggregated_features(df, feature, months, reference_date):\n    # Filter the DataFrame to include only the last 'months' months\n    filtered_df = df.filter(pl.col('S_2') >= (reference_date - pl.duration(days=30*months)))\n    \n    # Group by 'customer_ID' and aggregate the feature\n    agg_df = filtered_df.group_by('customer_ID').agg([\n        (pl.col(feature).sum() / pl.count(feature)).alias(f\"{feature}_Response_Rate_{months}\"),\n        (pl.col(feature).sum().map_elements(lambda x: 1 if x > 0 else 0)).alias(f\"{feature}_Ever_Response_{months}\")\n    ])\n    \n    return agg_df\n\n# Initialize a list to store the feature DataFrames\ncat_feature_dfs = []\n\n# Iterate over the categorical columns and create aggregated features\nfor feature in df_cat.columns:\n    if feature in ['customer_ID', 'S_2']:\n        continue\n    for months in [6, 12]:\n        cat_feature_dfs.append(create_categorical_aggregated_features(df_cat, feature, months, reference_date))\n\n# Join all features into a single DataFrame\n# Remove 'customer_ID' from all but the first DataFrame to avoid the DuplicateError\nfor i in range(1, len(cat_feature_dfs)):\n    cat_feature_dfs[i] = cat_feature_dfs[i].drop('customer_ID')\n\nfinal_cat_df = pl.concat(cat_feature_dfs, how='horizontal')\n\n# print(final_cat_df)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:09:15.996208Z","iopub.execute_input":"2024-04-10T23:09:15.996903Z","iopub.status.idle":"2024-04-10T23:09:29.217784Z","shell.execute_reply.started":"2024-04-10T23:09:15.996870Z","shell.execute_reply":"2024-04-10T23:09:29.216444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = pl.concat([final_features_df, final_cat_df.drop(\"customer_ID\"), development_sample.select(['customer_ID', 'S_2', 'target'] + continuous_columns).group_by('customer_ID').first().drop(\"customer_ID\")], how = \"horizontal\")","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:09:36.853267Z","iopub.execute_input":"2024-04-10T23:09:36.853716Z","iopub.status.idle":"2024-04-10T23:09:37.051165Z","shell.execute_reply.started":"2024-04-10T23:09:36.853681Z","shell.execute_reply":"2024-04-10T23:09:37.050153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:09:53.077788Z","iopub.execute_input":"2024-04-10T23:09:53.078225Z","iopub.status.idle":"2024-04-10T23:09:53.085096Z","shell.execute_reply.started":"2024-04-10T23:09:53.078192Z","shell.execute_reply":"2024-04-10T23:09:53.084003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature selection using XGB","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.impute import SimpleImputer","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:09:58.004985Z","iopub.execute_input":"2024-04-10T23:09:58.006106Z","iopub.status.idle":"2024-04-10T23:09:58.447136Z","shell.execute_reply.started":"2024-04-10T23:09:58.006059Z","shell.execute_reply":"2024-04-10T23:09:58.446091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(dataset.drop(['customer_ID', 'target']), dataset['target'], test_size = 0.3, random_state = 6)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:10:01.341533Z","iopub.execute_input":"2024-04-10T23:10:01.341959Z","iopub.status.idle":"2024-04-10T23:10:02.107657Z","shell.execute_reply.started":"2024-04-10T23:10:01.341909Z","shell.execute_reply":"2024-04-10T23:10:02.106649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute missing values with NaN\nimputer = SimpleImputer(strategy='constant', fill_value=np.nan)\nX_train_imputed = imputer.fit_transform(X_train)\nX_test_imputed = imputer.transform(X_test)\n\n# Convert the imputed arrays back to DataFrames\nX_train_imputed = pd.DataFrame(X_train_imputed, columns=X_train.columns)\nX_test_imputed = pd.DataFrame(X_test_imputed, columns=X_test.columns)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:10:06.929355Z","iopub.execute_input":"2024-04-10T23:10:06.930239Z","iopub.status.idle":"2024-04-10T23:10:11.862525Z","shell.execute_reply.started":"2024-04-10T23:10:06.930195Z","shell.execute_reply":"2024-04-10T23:10:11.861316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_test['S_2'])","metadata":{"execution":{"iopub.status.busy":"2024-04-11T00:13:44.765737Z","iopub.execute_input":"2024-04-11T00:13:44.766212Z","iopub.status.idle":"2024-04-11T00:13:44.773822Z","shell.execute_reply.started":"2024-04-11T00:13:44.766152Z","shell.execute_reply":"2024-04-11T00:13:44.772359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_1, X_test_2, y_test_1, y_test_2 = train_test_split(X_test_imputed, y_test, test_size = 0.5, random_state = 6)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:10:14.326787Z","iopub.execute_input":"2024-04-10T23:10:14.327687Z","iopub.status.idle":"2024-04-10T23:10:14.693334Z","shell.execute_reply.started":"2024-04-10T23:10:14.327590Z","shell.execute_reply":"2024-04-10T23:10:14.692056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier = xgb.XGBClassifier(seed=6)\nclassifier.fit(X_train_imputed, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:10:18.335470Z","iopub.execute_input":"2024-04-10T23:10:18.335902Z","iopub.status.idle":"2024-04-10T23:12:19.517386Z","shell.execute_reply.started":"2024-04-10T23:10:18.335866Z","shell.execute_reply":"2024-04-10T23:12:19.515364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier.feature_importances_","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:12:37.548570Z","iopub.execute_input":"2024-04-10T23:12:37.549379Z","iopub.status.idle":"2024-04-10T23:12:37.565481Z","shell.execute_reply.started":"2024-04-10T23:12:37.549332Z","shell.execute_reply":"2024-04-10T23:12:37.564043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get feature importances\nfeature_importances = {key: value for key, value in zip(dataset.drop(['customer_ID', 'target']).columns, classifier.feature_importances_)}\n\n# Filter features with importance higher than 0.5%\nimportant_features = {k: v for k, v in feature_importances.items() if v > 0.005}\n\n# Save the feature importances to a CSV file\nimportance_csv = pd.DataFrame.from_dict(important_features, orient='index', columns=['Importance'])\nimportance_csv.to_csv(\"feat_o.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:12:50.179565Z","iopub.execute_input":"2024-04-10T23:12:50.180000Z","iopub.status.idle":"2024-04-10T23:12:50.205270Z","shell.execute_reply.started":"2024-04-10T23:12:50.179964Z","shell.execute_reply":"2024-04-10T23:12:50.203994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(important_features)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:12:55.038644Z","iopub.execute_input":"2024-04-10T23:12:55.039636Z","iopub.status.idle":"2024-04-10T23:12:55.045925Z","shell.execute_reply.started":"2024-04-10T23:12:55.039511Z","shell.execute_reply":"2024-04-10T23:12:55.044537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(importance_csv.head(20))","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:12:57.950224Z","iopub.execute_input":"2024-04-10T23:12:57.950607Z","iopub.status.idle":"2024-04-10T23:12:57.960014Z","shell.execute_reply.started":"2024-04-10T23:12:57.950578Z","shell.execute_reply":"2024-04-10T23:12:57.958165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Train the XGBoost classifier with specified parameters\nclassifier = xgb.XGBClassifier(n_estimators=300, learning_rate=0.5, max_depth=4, subsample=0.5, colsample_bytree=0.5, scale_pos_weight=5, seed=6)\nclassifier.fit(X_train_imputed, y_train)\n\n# Get feature importances\nfeature_importances = {key: value for key, value in zip(X_train.columns, classifier.feature_importances_)}\n\n# Filter features with importance higher than 0.5%\nimportant_features_2 = {k: v for k, v in feature_importances.items() if v > 0.005}\n\n# Save the feature importances to a CSV file\nimportance_csv_2 = pd.DataFrame.from_dict(important_features_2, orient='index', columns=['Importance'])\nimportance_csv_2.to_csv(\"feat_o_model_2.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:13:02.761187Z","iopub.execute_input":"2024-04-10T23:13:02.762083Z","iopub.status.idle":"2024-04-10T23:16:27.156560Z","shell.execute_reply.started":"2024-04-10T23:13:02.762045Z","shell.execute_reply":"2024-04-10T23:16:27.154891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(important_features_2)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:16:50.580856Z","iopub.execute_input":"2024-04-10T23:16:50.581306Z","iopub.status.idle":"2024-04-10T23:16:50.587006Z","shell.execute_reply.started":"2024-04-10T23:16:50.581273Z","shell.execute_reply":"2024-04-10T23:16:50.585887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(importance_csv_2.head(10))","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:16:51.998269Z","iopub.execute_input":"2024-04-10T23:16:51.998678Z","iopub.status.idle":"2024-04-10T23:16:52.008393Z","shell.execute_reply.started":"2024-04-10T23:16:51.998644Z","shell.execute_reply":"2024-04-10T23:16:52.006857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_important_features = list(set(important_features.keys()) | set(important_features_2.keys()))","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:16:57.096187Z","iopub.execute_input":"2024-04-10T23:16:57.096616Z","iopub.status.idle":"2024-04-10T23:16:57.102879Z","shell.execute_reply.started":"2024-04-10T23:16:57.096582Z","shell.execute_reply":"2024-04-10T23:16:57.101748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(combined_important_features)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:17:28.037719Z","iopub.execute_input":"2024-04-10T23:17:28.038157Z","iopub.status.idle":"2024-04-10T23:17:28.043877Z","shell.execute_reply.started":"2024-04-10T23:17:28.038121Z","shell.execute_reply":"2024-04-10T23:17:28.042763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Combine the feature importances from both models\nall_feature_importances = {**important_features, **important_features_2}\n\n# Sort the features by their importance\nsorted_features = sorted(all_feature_importances.items(), key=lambda x: x[1], reverse=True)\n\n# Separate the feature names and their importances\nfeature_names, importances = zip(*sorted_features)\n\n# Create a bar plot\nplt.figure(figsize=(10, 8))\nplt.barh(feature_names, importances)\nplt.xlabel('Feature Importance')\nplt.ylabel('Feature')\nplt.title('Feature Selection Process')\nplt.gca().invert_yaxis()  # Invert y-axis to have the most important feature on top\nplt.show()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:17:34.559138Z","iopub.execute_input":"2024-04-10T23:17:34.559619Z","iopub.status.idle":"2024-04-10T23:17:34.931661Z","shell.execute_reply.started":"2024-04-10T23:17:34.559581Z","shell.execute_reply":"2024-04-10T23:17:34.930204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:17:47.737574Z","iopub.execute_input":"2024-04-10T23:17:47.738323Z","iopub.status.idle":"2024-04-10T23:17:47.742927Z","shell.execute_reply.started":"2024-04-10T23:17:47.738290Z","shell.execute_reply":"2024-04-10T23:17:47.742024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the parameter grid with your specified parameters\nparam_grid = {\n    'n_estimators': [50, 100, 300],\n    'learning_rate': [0.01, 0.1],\n    'subsample': [0.5, 0.8],\n    'colsample_bytree': [0.5, 1.0],\n    'scale_pos_weight': [1, 5, 10]\n}\n\n# Create an empty list to store the results\nresults = []\n\n# Iterate through each combination of parameters\nfor n_estimators in param_grid['n_estimators']:\n    for learning_rate in param_grid['learning_rate']:\n        for subsample in param_grid['subsample']:\n            for colsample_bytree in param_grid['colsample_bytree']:\n                for scale_pos_weight in param_grid['scale_pos_weight']:\n                    # Train a model with the current combination of parameters\n                    model = xgb.XGBClassifier(\n                        n_estimators=n_estimators,\n                        learning_rate=learning_rate,\n                        subsample=subsample,\n                        colsample_bytree=colsample_bytree,\n                        scale_pos_weight=scale_pos_weight,\n                        random_state=42\n                    )\n                    model.fit(X_train[combined_important_features], y_train)\n\n                    # Compute AUC scores on Train and Test datasets\n                    auc_train = roc_auc_score(y_train, model.predict_proba(X_train[combined_important_features])[:, 1])\n                    auc_test1 = roc_auc_score(y_test_1, model.predict_proba(X_test_1[combined_important_features])[:, 1])\n                    auc_test2 = roc_auc_score(y_test_2, model.predict_proba(X_test_2[combined_important_features])[:, 1])\n\n                    # Append the results to the list\n                    results.append({\n                        'Number of Trees': n_estimators,\n                        'Learning Rate': learning_rate,\n                        'Subsample': subsample,\n                        'Percentage of Features': colsample_bytree * 100,\n                        'Weight of Default': scale_pos_weight,\n                        'AUC Train': auc_train,\n                        'AUC Test 1': auc_test1,\n                        'AUC Test 2': auc_test2\n                    })\n\n# Convert the list of results to a DataFrame\nresults_df = pd.DataFrame(results)\nprint(results_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:17:50.973768Z","iopub.execute_input":"2024-04-10T23:17:50.974445Z","iopub.status.idle":"2024-04-10T23:19:22.282720Z","shell.execute_reply.started":"2024-04-10T23:17:50.974411Z","shell.execute_reply":"2024-04-10T23:19:22.281410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort the results by AUC Test 1 and AUC Test 2 (descending order)\nsorted_results_df = results_df.sort_values(by=['AUC Test 1', 'AUC Test 2'], ascending=False)\n\n# Print the sorted table\nprint(sorted_results_df)\n\n# Select the top row as the best model\nbest_model_row = sorted_results_df.iloc[0]\nprint(\"Best model parameters:\")\nprint(best_model_row)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:19:47.325646Z","iopub.execute_input":"2024-04-10T23:19:47.326459Z","iopub.status.idle":"2024-04-10T23:19:47.346089Z","shell.execute_reply.started":"2024-04-10T23:19:47.326422Z","shell.execute_reply":"2024-04-10T23:19:47.345011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming best_model_row contains the best parameters\noptimum_params = {\n    'n_estimators': int(best_model_row['Number of Trees']),\n    'learning_rate': best_model_row['Learning Rate'],\n    'subsample': best_model_row['Subsample'],\n    'colsample_bytree': best_model_row['Percentage of Features'] / 100,\n    'scale_pos_weight': best_model_row['Weight of Default'],\n    'random_state': 42\n}\n\n# Re-train the model with the optimum parameters\nfinal_model_xgb = xgb.XGBClassifier(**optimum_params)\nfinal_model_xgb.fit(X_train[combined_important_features], y_train)\n\n# Save the final model\nimport joblib\njoblib.dump(final_model_xgb, 'final_xgb_model.joblib')\n\nprint(\"Final model saved as 'final_xgb_model.joblib'\")","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:19:52.926680Z","iopub.execute_input":"2024-04-10T23:19:52.927348Z","iopub.status.idle":"2024-04-10T23:19:55.089925Z","shell.execute_reply.started":"2024-04-10T23:19:52.927313Z","shell.execute_reply":"2024-04-10T23:19:55.088534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(optimum_params)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:19:57.716012Z","iopub.execute_input":"2024-04-10T23:19:57.716418Z","iopub.status.idle":"2024-04-10T23:19:57.722251Z","shell.execute_reply.started":"2024-04-10T23:19:57.716388Z","shell.execute_reply":"2024-04-10T23:19:57.721068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate average and standard deviation of AUC across three samples for each model\navg_auc = []\nstd_auc = []\nauc_train = []\nauc_test2 = []\n\nfor idx, row in sorted_results_df.iterrows():\n    auc_train.append(row['AUC Train'])\n    auc_test2.append(row['AUC Test 2'])\n    \n    avg = (row['AUC Train'] + row['AUC Test 1'] + row['AUC Test 2']) / 3\n    avg_auc.append(avg)\n    std = ((row['AUC Train'] - avg)**2 + (row['AUC Test 1'] - avg)**2 + (row['AUC Test 2'] - avg)**2 / 3)**0.5\n    std_auc.append(std)\n\n# Create scatter plots\n\nplt.figure(figsize=(12, 6))\n\n# Scatter plot for Average AUC vs. Standard Deviation of AUC\nplt.subplot(1, 2, 1)\nplt.scatter(avg_auc, std_auc, color='b', alpha=0.7)\nplt.xlabel('Average AUC')\nplt.ylabel('Standard Deviation of AUC')\nplt.title('Average AUC vs. Standard Deviation of AUC')\n\n# Scatter plot for AUC of train sample vs. AUC of Test 2 sample\nplt.subplot(1, 2, 2)\nplt.scatter(auc_train, auc_test2, color='r', alpha=0.7)\nplt.xlabel('AUC of Train Sample')\nplt.ylabel('AUC of Test 2 Sample')\nplt.title('AUC of Train Sample vs. AUC of Test 2 Sample')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:20:01.287234Z","iopub.execute_input":"2024-04-10T23:20:01.287647Z","iopub.status.idle":"2024-04-10T23:20:01.874130Z","shell.execute_reply.started":"2024-04-10T23:20:01.287618Z","shell.execute_reply":"2024-04-10T23:20:01.873238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_results_df.to_csv('sorted_results.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:20:07.487652Z","iopub.execute_input":"2024-04-10T23:20:07.488586Z","iopub.status.idle":"2024-04-10T23:20:07.497715Z","shell.execute_reply.started":"2024-04-10T23:20:07.488546Z","shell.execute_reply":"2024-04-10T23:20:07.496486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict probabilities of default for train and test samples\ntrain_scores = final_model_xgb.predict_proba(X_train[combined_important_features])[:, 1]\ntest1_scores = final_model_xgb.predict_proba(X_test_1[combined_important_features])[:, 1]\ntest2_scores = final_model_xgb.predict_proba(X_test_2[combined_important_features])[:, 1]\n\nnum_bins = 10\ntrain_scores_sorted = np.sort(train_scores)\nbins = np.linspace(train_scores_sorted[0], train_scores_sorted[-1], num_bins+1)\n\n# Calculate default rates for each bin\ndef calc_default_rates(scores, y, bins):\n    df = pd.DataFrame({'Score': scores, 'Default': y})\n    df['Bin'] = pd.cut(df['Score'], bins=bins, include_lowest=True)\n    default_rates = df.groupby('Bin')['Default'].mean()\n    return default_rates\n\ntrain_default_rates = calc_default_rates(train_scores, y_train, bins)\ntest1_default_rates = calc_default_rates(test1_scores, y_test_1, bins)\ntest2_default_rates = calc_default_rates(test2_scores, y_test_2, bins)\n\n# Create a bar chart\nbar_width = 0.25\nindex = np.arange(num_bins)\n\nplt.bar(index, train_default_rates, bar_width, label='Train')\nplt.bar(index + bar_width, test1_default_rates, bar_width, label='Test 1')\nplt.bar(index + 2 * bar_width, test2_default_rates, bar_width, label='Test 2')\n\nplt.xlabel('Score Bins')\nplt.ylabel('Default Rate')\nplt.xticks(index + bar_width, [f'Bin {i+1}' for i in range(num_bins)], rotation=45)\nplt.legend()\nplt.title('Default Rates by Score Bins')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:20:19.959576Z","iopub.execute_input":"2024-04-10T23:20:19.960380Z","iopub.status.idle":"2024-04-10T23:20:20.854597Z","shell.execute_reply.started":"2024-04-10T23:20:19.960328Z","shell.execute_reply":"2024-04-10T23:20:20.853661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print bin values and default rates\nprint(\"Bins and Default Rates:\")\nprint(\"Bin Range\\t\\tTrain\\t\\tTest 1\\t\\tTest 2\")\nfor i in range(num_bins):\n    bin_range = f\"[{bins[i]:.2f}, {bins[i+1]:.2f})\"\n    train_rate = train_default_rates.iloc[i]\n    test1_rate = test1_default_rates.iloc[i]\n    test2_rate = test2_default_rates.iloc[i]\n    print(f\"{bin_range}\\t{train_rate:.4f}\\t{test1_rate:.4f}\\t{test2_rate:.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:20:25.799450Z","iopub.execute_input":"2024-04-10T23:20:25.800218Z","iopub.status.idle":"2024-04-10T23:20:25.807707Z","shell.execute_reply.started":"2024-04-10T23:20:25.800178Z","shell.execute_reply":"2024-04-10T23:20:25.806479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shap\n\n# Compute SHAP values\nexplainer = shap.Explainer(final_model_xgb)\nshap_values = explainer(X_test_2[combined_important_features])\n\n# Create a Beeswarm plot\nshap.summary_plot(shap_values, X_test_2[combined_important_features], plot_type=\"dot\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:20:32.228329Z","iopub.execute_input":"2024-04-10T23:20:32.229049Z","iopub.status.idle":"2024-04-10T23:21:07.281506Z","shell.execute_reply.started":"2024-04-10T23:20:32.229006Z","shell.execute_reply":"2024-04-10T23:21:07.280394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select an observation (e.g., the first observation in Test 2 sample)\nobservation_index = 0\nselected_observation = X_test_2[combined_important_features].iloc[observation_index]\n\n# Add an extra dimension to the selected observation to make it 2D\nselected_observation_2d = selected_observation.values.reshape(1, -1)\n\n# Compute SHAP values for the selected observation\nshap_values_observation = explainer(selected_observation_2d)\n\n# Set the feature names in the Explanation object\nshap_values_observation.feature_names = combined_important_features\n\n# Create a Waterfall plot\nshap.waterfall_plot(shap_values_observation[0])\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:21:18.094910Z","iopub.execute_input":"2024-04-10T23:21:18.095527Z","iopub.status.idle":"2024-04-10T23:21:18.933108Z","shell.execute_reply.started":"2024-04-10T23:21:18.095495Z","shell.execute_reply":"2024-04-10T23:21:18.931710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Network","metadata":{}},{"cell_type":"markdown","source":"## Missing Value replacement with 0","metadata":{}},{"cell_type":"code","source":"# Replace NaN values with 0 in the imputed DataFrames\nX_train_nn = X_train_imputed.fillna(0)\nX_test_1_nn = X_test_1.fillna(0)\nX_test_2_nn = X_test_2.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T01:19:06.874878Z","iopub.execute_input":"2024-04-11T01:19:06.875355Z","iopub.status.idle":"2024-04-11T01:19:08.450702Z","shell.execute_reply.started":"2024-04-11T01:19:06.875313Z","shell.execute_reply":"2024-04-11T01:19:08.449173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outlier Treatment","metadata":{}},{"cell_type":"code","source":"# Calculate the 1st and 99th percentiles\npercentiles = X_train_nn.quantile([0.01, 0.99])\n\n# Cap and floor the values\nX_train_nn = X_train_nn.clip(percentiles.iloc[0], percentiles.iloc[1], axis=1)\nX_test_1_nn = X_test_1_nn.clip(percentiles.iloc[0], percentiles.iloc[1], axis=1)\nX_test_2_nn = X_test_2_nn.clip(percentiles.iloc[0], percentiles.iloc[1], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T01:19:11.571532Z","iopub.execute_input":"2024-04-11T01:19:11.572209Z","iopub.status.idle":"2024-04-11T01:19:15.351566Z","shell.execute_reply.started":"2024-04-11T01:19:11.572169Z","shell.execute_reply":"2024-04-11T01:19:15.350413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalization","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Initialize the scaler\nscaler = StandardScaler()\n\n# Fit the scaler on the train data and transform both train and test data\nX_train_nn = scaler.fit_transform(X_train_nn)\nX_test_1_nn = scaler.transform(X_test_1_nn)\nX_test_2_nn = scaler.transform(X_test_2_nn)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T01:19:22.268930Z","iopub.execute_input":"2024-04-11T01:19:22.269351Z","iopub.status.idle":"2024-04-11T01:19:23.875819Z","shell.execute_reply.started":"2024-04-11T01:19:22.269321Z","shell.execute_reply":"2024-04-11T01:19:23.874649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.metrics import roc_auc_score\nimport pandas as pd\n\nresults_df = pd.DataFrame(columns=['HL', 'Node', 'Activation Function', 'Dropout', 'Batch Size', 'AUC Train', 'AUC Test1', 'AUC Test2'])\n\n# Hyperparameters\nn_layers_values = [2, 4]\nn_nodes_values = [4, 6]\nactivation_functions = ['relu', 'tanh']\ndropout_rates = [0.5, 1.0]  # 50% dropout and no dropout\nbatch_sizes = [100, 10000]\nepochs = 20\n\n# Loop over each combination of hyperparameters\nfor n_layers in n_layers_values:\n    for n_node in n_nodes_values:\n        for activation in activation_functions:\n            for dropout in dropout_rates:\n                for batch_size in batch_sizes:\n                    # Build and compile the model\n                    model = tf.keras.models.Sequential()\n                    model.add(tf.keras.layers.Input(shape=(X_train.shape[1],)))  # Input layer\n                    for _ in range(n_layers):\n                        model.add(tf.keras.layers.Dense(n_node, activation=activation))\n                        if dropout < 1.0:\n                            model.add(tf.keras.layers.Dropout(dropout))\n                    model.add(tf.keras.layers.Dense(1, activation='sigmoid'))  # Output layer\n                    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[tf.keras.metrics.AUC()])\n\n                    # Train the model\n                    history = model.fit(X_train_nn, y_train, epochs=epochs, batch_size=batch_size, verbose=0)\n\n                    # Calculate ROC AUC scores for train, test1, and test2 sets\n                    auc_train = roc_auc_score(y_train, model.predict(X_train_nn))\n                    auc_test1 = roc_auc_score(y_test_1, model.predict(X_test_1_nn))\n                    auc_test2 = roc_auc_score(y_test_2, model.predict(X_test_2_nn))\n\n                    # Create a DataFrame from the results\n                    result_dict = {\n                        'HL': n_layers,\n                        'Node': n_node,\n                        'Activation Function': activation,\n                        'Dropout': f\"{int((1 - dropout) * 100)}%\",\n                        'Batch Size': batch_size,\n                        'AUC Train': auc_train,\n                        'AUC Test1': auc_test1,\n                        'AUC Test2': auc_test2\n                    }\n\n                    result_df = pd.DataFrame([result_dict])\n\n                    # Concatenate the DataFrame to results_df\n                    results_df = pd.concat([results_df, result_df], ignore_index=True)\n\n                    print(f\"Model with {n_layers} layers, {n_node} nodes per layer, {activation} activation, \"\n                          f\"{int((1 - dropout) * 100)}% dropout, and batch size {batch_size} finished training. \"\n                          f\"Train AUC: {auc_train}, Test1 AUC: {auc_test1}, Test2 AUC: {auc_test2}\")\n\n# Optionally, save the results to a CSV file after the entire grid search\nresults_df.to_csv('grid_search_results.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T01:19:26.809990Z","iopub.execute_input":"2024-04-11T01:19:26.810395Z","iopub.status.idle":"2024-04-11T01:39:27.847244Z","shell.execute_reply.started":"2024-04-11T01:19:26.810366Z","shell.execute_reply":"2024-04-11T01:39:27.846026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort the results by AUC Test 1 and AUC Test 2 (descending order)\nsorted_results_df_2 = results_df.sort_values(by=['AUC Test1', 'AUC Test2'], ascending=False)\n\n# Print the sorted table\nprint(sorted_results_df_2)\n\nsorted_results_df_2.to_csv('sorted_results_df_2.csv', index = False)\n\n# Select the top row as the best model\nbest_model_row_2 = sorted_results_df_2.iloc[0]\nprint(\"Best model parameters:\")\nprint(best_model_row_2)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:17:23.530084Z","iopub.execute_input":"2024-04-11T02:17:23.530531Z","iopub.status.idle":"2024-04-11T02:17:23.553321Z","shell.execute_reply.started":"2024-04-11T02:17:23.530497Z","shell.execute_reply":"2024-04-11T02:17:23.552048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout, BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras.regularizers import l2\n\noptimum_params = {\n    'n_layers': int(best_model_row_2['HL']),\n    'n_nodes': int(best_model_row_2['Node']),\n    'activation': best_model_row_2['Activation Function'],\n    'dropout_rate': 1 - (int(best_model_row_2['Dropout'].strip('%')) / 100),\n    'batch_size': int(best_model_row_2['Batch Size']),\n}\n\n\nfinal_model = Sequential()\nfinal_model.add(Dense(optimum_params['n_nodes'], input_dim=X_train.shape[1], activation=optimum_params['activation'], kernel_regularizer=l2(0.01)))\nfor _ in range(optimum_params['n_layers'] - 1):\n    final_model.add(Dense(optimum_params['n_nodes'], activation=optimum_params['activation'], kernel_regularizer=l2(0.01)))\n    final_model.add(BatchNormalization())\n    if optimum_params['dropout_rate'] < 1:\n        final_model.add(Dropout(optimum_params['dropout_rate']))\nfinal_model.add(Dense(1, activation='sigmoid'))\n\noptimizer = Adam(learning_rate=0.001, clipnorm=1.0)\nfinal_model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=[tf.keras.metrics.AUC()])\n\nfinal_model.fit(X_train_nn, y_train, epochs=20, batch_size=optimum_params['batch_size'], verbose=1, validation_data=(X_test_1_nn, y_test_1))\n\n# Save the model\nfinal_model.save('final_nn_model.keras')","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:17:28.929073Z","iopub.execute_input":"2024-04-11T02:17:28.929479Z","iopub.status.idle":"2024-04-11T02:18:21.472571Z","shell.execute_reply.started":"2024-04-11T02:17:28.929450Z","shell.execute_reply":"2024-04-11T02:18:21.471045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the best model saved during training\nfinal_model.load_weights('final_nn_model.keras')\n\nprint(\"Final NN model saved as 'final_nn_model.keras'\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:19:21.145109Z","iopub.execute_input":"2024-04-11T02:19:21.146468Z","iopub.status.idle":"2024-04-11T02:19:21.204492Z","shell.execute_reply.started":"2024-04-11T02:19:21.146412Z","shell.execute_reply":"2024-04-11T02:19:21.203072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Best Model","metadata":{}},{"cell_type":"code","source":"# Calculate average and standard deviation of AUC across three samples for each model\navg_auc = []\nstd_auc = []\nauc_train = []\nauc_test2 = []\n\nfor idx, row in sorted_results_df_2.iterrows():\n    auc_train.append(row['AUC Train'])\n    auc_test2.append(row['AUC Test2'])\n    \n    avg = (row['AUC Train'] + row['AUC Test1'] + row['AUC Test2']) / 3\n    avg_auc.append(avg)\n    std = ((row['AUC Train'] - avg)**2 + (row['AUC Test1'] - avg)**2 + (row['AUC Test2'] - avg)**2 / 3)**0.5\n    std_auc.append(std)\n\n# Create scatter plots\n\nplt.figure(figsize=(12, 6))\n\n# Scatter plot for Average AUC vs. Standard Deviation of AUC\nplt.subplot(1, 2, 1)\nplt.scatter(avg_auc, std_auc, color='b', alpha=0.7)\nplt.xlabel('Average AUC')\nplt.ylabel('Standard Deviation of AUC')\nplt.title('Average AUC vs. Standard Deviation of AUC')\n\n# Scatter plot for AUC of train sample vs. AUC of Test 2 sample\nplt.subplot(1, 2, 2)\nplt.scatter(auc_train, auc_test2, color='r', alpha=0.7)\nplt.xlabel('AUC of Train Sample')\nplt.ylabel('AUC of Test 2 Sample')\nplt.title('AUC of Train Sample vs. AUC of Test 2 Sample')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:36:11.636218Z","iopub.execute_input":"2024-04-11T02:36:11.636738Z","iopub.status.idle":"2024-04-11T02:36:13.479924Z","shell.execute_reply.started":"2024-04-11T02:36:11.636703Z","shell.execute_reply":"2024-04-11T02:36:13.478756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\n# Get predicted probabilities for the default class (1) for each sample\nnn_probs_train = final_model.predict(X_train_nn).flatten()\nnn_probs_test1 = final_model.predict(X_test_1_nn).flatten()\nnn_probs_test2 = final_model.predict(X_test_2_nn).flatten()\n\n# Define score bins based on the train sample\nbins = pd.qcut(nn_probs_train, q=10, duplicates='drop').categories\n\n# Apply the same thresholds to test samples\ntrain_bins = pd.cut(nn_probs_train, bins=bins, labels=bins, include_lowest=True)\ntest1_bins = pd.cut(nn_probs_test1, bins=bins, labels=bins, include_lowest=True)\ntest2_bins = pd.cut(nn_probs_test2, bins=bins, labels=bins, include_lowest=True)\n\n# Calculate default rate for each bin in each sample\ndef calculate_default_rates(y, bins):\n    df = pd.DataFrame({'y': y, 'bins': bins})\n    default_rates = df.groupby('bins')['y'].mean()\n    return default_rates\n\ntrain_default_rates = calculate_default_rates(y_train, train_bins)\ntest1_default_rates = calculate_default_rates(y_test_1, test1_bins)\ntest2_default_rates = calculate_default_rates(y_test_2, test2_bins)\n\n# Create a bar chart showing the default rate for each bin across the three samples\nwidth = 0.2  # width of the bars\nx = np.arange(len(bins))  # the label locations\n\nplt.bar(x - width, train_default_rates, width, label='Train')\nplt.bar(x, test1_default_rates, width, label='Test 1')\nplt.bar(x + width, test2_default_rates, width, label='Test 2')\n\nplt.xticks(x, bins, rotation=45)\nplt.xlabel('Score Bins')\nplt.ylabel('Default Rate')\nplt.title('Default Rate by Score Bins for Neural Network Model')\nplt.legend()\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:46:27.002331Z","iopub.execute_input":"2024-04-11T02:46:27.003909Z","iopub.status.idle":"2024-04-11T02:46:34.514232Z","shell.execute_reply.started":"2024-04-11T02:46:27.003850Z","shell.execute_reply":"2024-04-11T02:46:34.513157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_model_row)\nprint(\"----------------------------------\")\nprint(best_model_row_2)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:19:34.017062Z","iopub.execute_input":"2024-04-11T02:19:34.017510Z","iopub.status.idle":"2024-04-11T02:19:34.025720Z","shell.execute_reply.started":"2024-04-11T02:19:34.017476Z","shell.execute_reply":"2024-04-11T02:19:34.024423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract AUC scores for XGBoost and Neural Network models\nxgb_auc_test1 = best_model_row['AUC Test 1']\nxgb_auc_test2 = best_model_row['AUC Test 2']\nnn_auc_test1 = best_model_row_2['AUC Test1']\nnn_auc_test2 = best_model_row_2['AUC Test2']\n\n# Compare the average AUC scores\navg_auc_xgb = (xgb_auc_test1 + xgb_auc_test2) / 2\navg_auc_nn = (nn_auc_test1 + nn_auc_test2) / 2\n\nprint(f\"Average AUC for XGBoost: {avg_auc_xgb}\")\nprint(f\"Average AUC for Neural Network: {avg_auc_nn}\")\n\n# Determine the best model\nif avg_auc_xgb > avg_auc_nn:\n    print(\"XGBoost is the best model.\")\nelse:\n    print(\"Neural Network is the best model.\")","metadata":{"execution":{"iopub.status.busy":"2024-04-11T02:19:43.421640Z","iopub.execute_input":"2024-04-11T02:19:43.422093Z","iopub.status.idle":"2024-04-11T02:19:43.430581Z","shell.execute_reply.started":"2024-04-11T02:19:43.422058Z","shell.execute_reply":"2024-04-11T02:19:43.429283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get predictions for the train dataset\nxgb_predictions_train = final_model_xgb.predict(X_train_imputed[combined_important_features])\n\n# Get predictions for the test1 dataset\nxgb_predictions_test1 = final_model_xgb.predict(X_test_1[combined_important_features])\n\n# Get predictions for the test2 dataset\nxgb_predictions_test2 = final_model_xgb.predict(X_test_2[combined_important_features])\n\n# Get predicted probabilities for the train dataset\nxgb_probs_train = final_model_xgb.predict_proba(X_train_imputed[combined_important_features])\n\n# Get predicted probabilities for the test1 dataset\nxgb_probs_test1 = final_model_xgb.predict_proba(X_test_1[combined_important_features])\n\n# Get predicted probabilities for the test2 dataset\nxgb_probs_test2 = final_model_xgb.predict_proba(X_test_2[combined_important_features])","metadata":{"execution":{"iopub.status.busy":"2024-04-10T23:29:00.789528Z","iopub.execute_input":"2024-04-10T23:29:00.790209Z","iopub.status.idle":"2024-04-10T23:29:01.561855Z","shell.execute_reply.started":"2024-04-10T23:29:00.790176Z","shell.execute_reply":"2024-04-10T23:29:01.560373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the thresholds\nthresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n\n# Initialize lists to store default rates, portfolio revenues, and the number of applicants for the training data\ndefault_rates_train = []\nportfolio_revenues_train = []\napplicant_counts_train = []\n\n# Convert string literals to date objects for Polars\nstart_date = pl.datetime(2017, 11, 1)\nend_date = pl.datetime(2018, 4, 30)\n\n# Filter the data to include only the last 6 months\nfiltered_data_train = X_train.filter((pl.col('S_2') >= start_date) & (pl.col('S_2') <= end_date))\n\n# Calculate the average balance for the filtered data\naverage_balance_train = filtered_data_train.select(pl.mean('B_3')).to_numpy()[0, 0]\n\n# Calculate the monthly revenue for 1 customer for the training data\nmonthly_revenue_train = average_balance_train * 0.02\n\n# Calculate the expected annual revenue for the training data over the next 12 months\nexpected_revenue_train = monthly_revenue_train * 12\n\n# Iterate through the thresholds for the training data\nfor threshold in thresholds:\n    # Filter applicants based on the threshold for the training data\n    accepted_indices_train = xgb_probs_train[:, 1] < threshold  # True for accepted, False for rejected\n\n    # Convert the NumPy boolean array to a Polars Series\n    accepted_indices_train_series = pl.Series(accepted_indices_train)\n\n    # Calculate the number of applicants (both defaulted and not defaulted) for the training data\n    total_applicants_train = np.sum(accepted_indices_train)\n\n    # Calculate the number of applicants who defaulted for the training data\n    defaulted_applicants_train = y_train.filter(accepted_indices_train_series).sum()\n\n    # Calculate the default rate among all applicants for the training data\n    default_rate_train = defaulted_applicants_train / total_applicants_train if total_applicants_train > 0 else 0\n\n    # Calculate the portfolio revenue for the training data\n    portfolio_revenue_value_train = np.sum(expected_revenue_train * accepted_indices_train)\n\n    # Append results to the lists for the training data\n    default_rates_train.append(default_rate_train)\n    portfolio_revenues_train.append(portfolio_revenue_value_train)\n    applicant_counts_train.append(total_applicants_train)\n\n# Print the results for each threshold for the training data\nfor i, threshold in enumerate(thresholds):\n    print(f\"Training Data - Threshold: {threshold:.1f}, Default Rate: {default_rates_train[i]:.2%}, Portfolio Revenue: ${portfolio_revenues_train[i]:.2f}, Number of Applicants: {applicant_counts_train[i]}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T00:01:46.814410Z","iopub.execute_input":"2024-04-11T00:01:46.815338Z","iopub.status.idle":"2024-04-11T00:01:46.876328Z","shell.execute_reply.started":"2024-04-11T00:01:46.815290Z","shell.execute_reply":"2024-04-11T00:01:46.874968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_1_new, X_test_2_new, y_test_1_new, y_test_2_new = train_test_split(X_test, y_test, test_size = 0.5, random_state = 6)","metadata":{"execution":{"iopub.status.busy":"2024-04-11T00:27:09.478472Z","iopub.execute_input":"2024-04-11T00:27:09.478966Z","iopub.status.idle":"2024-04-11T00:27:09.814451Z","shell.execute_reply.started":"2024-04-11T00:27:09.478914Z","shell.execute_reply":"2024-04-11T00:27:09.813361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the thresholds\nthresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n\n# Initialize lists to store default rates, portfolio revenues, and the number of applicants for the training data\ndefault_rates_test_1 = []\nportfolio_revenues_test_1 = []\napplicant_counts_test_1 = []\n\n# Convert string literals to date objects for Polars\nstart_date = pl.datetime(2017, 11, 1)\nend_date = pl.datetime(2018, 4, 30)\n\n# Filter the data to include only the last 6 months\nfiltered_data_test_1 = X_test_1_new.filter((pl.col('S_2') >= start_date) & (pl.col('S_2') <= end_date))\n\n# Calculate the average balance for the filtered data\naverage_balance_test_1 = filtered_data_test_1.select(pl.mean('B_3')).to_numpy()[0, 0]\n\n# Calculate the monthly revenue for 1 customer for the training data\nmonthly_revenue_test_1 = average_balance_test_1 * 0.02\n\n# Calculate the expected annual revenue for the training data over the next 12 months\nexpected_revenue_test_1 = monthly_revenue_test_1 * 12\n\n# Iterate through the thresholds for the training data\nfor threshold in thresholds:\n    # Filter applicants based on the threshold for the training data\n    accepted_indices_test_1 = xgb_probs_test1[:, 1] < threshold  # True for accepted, False for rejected\n\n    # Convert the NumPy boolean array to a Polars Series\n    accepted_indices_test1_series = pl.Series(accepted_indices_test_1)\n\n    # Calculate the number of applicants (both defaulted and not defaulted) for the training data\n    total_applicants_test_1 = np.sum(accepted_indices_test_1)\n\n    # Calculate the number of applicants who defaulted for the training data\n    defaulted_applicants_test_1 = y_test_1_new.filter(accepted_indices_test1_series).sum()\n\n    # Calculate the default rate among all applicants for the training data\n    default_rate_test_1 = defaulted_applicants_test_1 / total_applicants_test_1 if total_applicants_test_1 > 0 else 0\n\n    # Calculate the portfolio revenue for the training data\n    portfolio_revenue_value_test_1 = np.sum(expected_revenue_test_1 * accepted_indices_test_1)\n\n    # Append results to the lists for the training data\n    default_rates_test_1.append(default_rate_test_1)\n    portfolio_revenues_test_1.append(portfolio_revenue_value_test_1)\n    applicant_counts_test_1.append(total_applicants_test_1)\n\n# Print the results for each threshold for the training data\nfor i, threshold in enumerate(thresholds):\n    print(f\"Test-1 Data - Threshold: {threshold:.1f}, Default Rate: {default_rates_test_1[i]:.2%}, Portfolio Revenue: ${portfolio_revenues_test_1[i]:.2f}, Number of Applicants: {applicant_counts_test_1[i]}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T00:37:02.926443Z","iopub.execute_input":"2024-04-11T00:37:02.926926Z","iopub.status.idle":"2024-04-11T00:37:02.959853Z","shell.execute_reply.started":"2024-04-11T00:37:02.926894Z","shell.execute_reply":"2024-04-11T00:37:02.958632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the thresholds\nthresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]\n\n# Initialize lists to store default rates, portfolio revenues, and the number of applicants for the training data\ndefault_rates_test_2 = []\nportfolio_revenues_test_2 = []\napplicant_counts_test_2 = []\n\n# Convert string literals to date objects for Polars\nstart_date = pl.datetime(2017, 11, 1)\nend_date = pl.datetime(2018, 4, 30)\n\n# Filter the data to include only the last 6 months\nfiltered_data_test_2 = X_test_2_new.filter((pl.col('S_2') >= start_date) & (pl.col('S_2') <= end_date))\n\n# Calculate the average balance for the filtered data\naverage_balance_test_2 = filtered_data_test_2.select(pl.mean('B_3')).to_numpy()[0, 0]\n\n# Calculate the monthly revenue for 1 customer for the training data\nmonthly_revenue_test_2 = average_balance_test_2 * 0.02\n\n# Calculate the expected annual revenue for the training data over the next 12 months\nexpected_revenue_test_2 = monthly_revenue_test_2 * 12\n\n# Iterate through the thresholds for the training data\nfor threshold in thresholds:\n    # Filter applicants based on the threshold for the training data\n    accepted_indices_test_2 = xgb_probs_test2[:, 1] < threshold  # True for accepted, False for rejected\n\n    # Convert the NumPy boolean array to a Polars Series\n    accepted_indices_test2_series = pl.Series(accepted_indices_test_2)\n\n    # Calculate the number of applicants (both defaulted and not defaulted) for the training data\n    total_applicants_test_2 = np.sum(accepted_indices_test_2)\n\n    # Calculate the number of applicants who defaulted for the training data\n    defaulted_applicants_test_2 = y_test_2_new.filter(accepted_indices_test2_series).sum()\n\n    # Calculate the default rate among all applicants for the training data\n    default_rate_test_2 = defaulted_applicants_test_2 / total_applicants_test_2 if total_applicants_test_2 > 0 else 0\n\n    # Calculate the portfolio revenue for the training data\n    portfolio_revenue_value_test_2 = np.sum(expected_revenue_test_2 * accepted_indices_test_2)\n\n    # Append results to the lists for the training data\n    default_rates_test_2.append(default_rate_test_2)\n    portfolio_revenues_test_2.append(portfolio_revenue_value_test_2)\n    applicant_counts_test_2.append(total_applicants_test_2)\n\n# Print the results for each threshold for the training data\nfor i, threshold in enumerate(thresholds):\n    print(f\"Test-2 Data - Threshold: {threshold:.1f}, Default Rate: {default_rates_test_2[i]:.2%}, Portfolio Revenue: ${portfolio_revenues_test_2[i]:.2f}, Number of Applicants: {applicant_counts_test_2[i]}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-11T00:41:12.327259Z","iopub.execute_input":"2024-04-11T00:41:12.327753Z","iopub.status.idle":"2024-04-11T00:41:12.358836Z","shell.execute_reply.started":"2024-04-11T00:41:12.327718Z","shell.execute_reply":"2024-04-11T00:41:12.357730Z"},"trusted":true},"execution_count":null,"outputs":[]}]}