{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7584174,"sourceType":"datasetVersion","datasetId":4414761}],"dockerImageVersionId":30683,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom xgboost import XGBClassifier\nimport optuna\nfrom sklearn.preprocessing import OneHotEncoder, MinMaxScaler, RobustScaler\nfrom sklearn.compose import make_column_transformer\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import LogisticRegression\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\nfrom imblearn.over_sampling import SMOTE\nfrom imblearn.ensemble import BalancedRandomForestClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split, cross_val_score, RepeatedStratifiedKFold\nfrom sklearn.metrics import confusion_matrix, f1_score, classification_report, roc_auc_score, recall_score, make_scorer, roc_curve, precision_score, accuracy_score\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\nimport joblib\n\nimport lightgbm as lgb\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-14T16:45:29.409022Z","iopub.execute_input":"2024-05-14T16:45:29.409763Z","iopub.status.idle":"2024-05-14T16:45:32.831514Z","shell.execute_reply.started":"2024-05-14T16:45:29.409722Z","shell.execute_reply":"2024-05-14T16:45:32.830274Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"### Pre-Fitted Voting Model","metadata":{}},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n def __init__(self, estimators):\n super().__init__()\n self.estimators = estimators\n \n def fit(self, X, y=None):\n return self\n \n def predict(self, X):\n y_preds = [estimator.predict(X) for estimator in self.estimators]\n return np.mean(y_preds, axis=0)\n \n def predict_proba(self, X):\n y_preds = [estimator.predict_proba(X) for estimator in self.estimators]\n return np.mean(y_preds, axis=0)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:35.63476Z","iopub.execute_input":"2024-05-14T16:45:35.63517Z","iopub.status.idle":"2024-05-14T16:45:35.643657Z","shell.execute_reply.started":"2024-05-14T16:45:35.635135Z","shell.execute_reply":"2024-05-14T16:45:35.642369Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"### Pipeline","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n @staticmethod\n def set_table_dtypes(df):\n for col in df.columns:\n if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n df = df.with_columns(pl.col(col).cast(pl.Int64))\n elif col in [\"date_decision\"]:\n df = df.with_columns(pl.col(col).cast(pl.Date))\n elif col[-1] in (\"P\", \"A\"):\n df = df.with_columns(pl.col(col).cast(pl.Float64))\n elif col[-1] in (\"M\",):\n df = df.with_columns(pl.col(col).cast(pl.String))\n elif col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col).cast(pl.Date)) \n\n return df\n \n @staticmethod\n def handle_dates(df):\n for col in df.columns:\n if col[-1] in (\"D\",):\n df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n df = df.with_columns(pl.col(col).dt.total_days())\n \n df = df.drop(\"date_decision\", \"MONTH\")\n\n return df\n \n @staticmethod\n def filter_cols(df):\n for col in df.columns:\n if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n isnull = df[col].is_null().mean()\n\n if isnull > 0.95:\n df = df.drop(col)\n\n for col in df.columns:\n if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n freq = df[col].n_unique()\n\n if (freq == 1) | (freq > 200):\n df = df.drop(col)\n\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:35.955785Z","iopub.execute_input":"2024-05-14T16:45:35.956746Z","iopub.status.idle":"2024-05-14T16:45:35.971084Z","shell.execute_reply.started":"2024-05-14T16:45:35.956708Z","shell.execute_reply":"2024-05-14T16:45:35.96994Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":"### Automatic Aggregation","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n @staticmethod\n def num_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n return expr_max\n\n @staticmethod\n def date_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n return expr_max\n\n @staticmethod\n def str_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"M\",)]\n \n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n return expr_max\n\n @staticmethod\n def other_expr(df):\n cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n \n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n return expr_max\n \n @staticmethod\n def count_expr(df):\n cols = [col for col in df.columns if \"num_group\" in col]\n\n expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n return expr_max\n\n @staticmethod\n def get_exprs(df):\n exprs = Aggregator.num_expr(df) + \\\n Aggregator.date_expr(df) + \\\n Aggregator.str_expr(df) + \\\n Aggregator.other_expr(df) + \\\n Aggregator.count_expr(df)\n\n return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:36.216972Z","iopub.execute_input":"2024-05-14T16:45:36.217697Z","iopub.status.idle":"2024-05-14T16:45:36.230589Z","shell.execute_reply.started":"2024-05-14T16:45:36.217659Z","shell.execute_reply":"2024-05-14T16:45:36.229405Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":"### File I/O","metadata":{}},{"cell_type":"code","source":"def read_file(path, depth=None):\n df = pl.read_parquet(path)\n df = df.pipe(Pipeline.set_table_dtypes)\n \n if depth in [1, 2]:\n df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n \n return df\n\ndef read_files(regex_path, depth=None):\n chunks = []\n for path in glob(str(regex_path)):\n chunks.append(pl.read_parquet(path).pipe(Pipeline.set_table_dtypes))\n \n df = pl.concat(chunks, how=\"vertical_relaxed\")\n if depth in [1, 2]:\n df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n \n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:36.496754Z","iopub.execute_input":"2024-05-14T16:45:36.497166Z","iopub.status.idle":"2024-05-14T16:45:36.505989Z","shell.execute_reply.started":"2024-05-14T16:45:36.497131Z","shell.execute_reply":"2024-05-14T16:45:36.504719Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n df_base = (\n df_base\n .with_columns(\n month_decision = pl.col(\"date_decision\").dt.month(),\n weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n )\n )\n \n for i, df in enumerate(depth_0 + depth_1 + depth_2):\n df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n \n df_base = df_base.pipe(Pipeline.handle_dates)\n \n return df_base","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:36.896665Z","iopub.execute_input":"2024-05-14T16:45:36.897447Z","iopub.status.idle":"2024-05-14T16:45:36.903996Z","shell.execute_reply.started":"2024-05-14T16:45:36.897389Z","shell.execute_reply":"2024-05-14T16:45:36.902954Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"def to_pandas(df_data, cat_cols=None):\n df_data = df_data.to_pandas()\n \n if cat_cols is None:\n cat_cols = list(df_data.select_dtypes(\"object\").columns)\n \n df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n \n return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:37.016319Z","iopub.execute_input":"2024-05-14T16:45:37.016733Z","iopub.status.idle":"2024-05-14T16:45:37.02299Z","shell.execute_reply.started":"2024-05-14T16:45:37.016699Z","shell.execute_reply":"2024-05-14T16:45:37.021836Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"markdown","source":"### Configuration","metadata":{}},{"cell_type":"code","source":"ROOT = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:37.287696Z","iopub.execute_input":"2024-05-14T16:45:37.288637Z","iopub.status.idle":"2024-05-14T16:45:37.293375Z","shell.execute_reply.started":"2024-05-14T16:45:37.288599Z","shell.execute_reply":"2024-05-14T16:45:37.292309Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":"### Train Files Read & Feature Engineering","metadata":{}},{"cell_type":"code","source":"data_store = {\n \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n \"depth_0\": [\n read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n ],\n \"depth_1\": [\n read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n ],\n \"depth_2\": [\n read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:45:37.554566Z","iopub.execute_input":"2024-05-14T16:45:37.55558Z","iopub.status.idle":"2024-05-14T16:46:11.294068Z","shell.execute_reply.started":"2024-05-14T16:45:37.555537Z","shell.execute_reply":"2024-05-14T16:46:11.29301Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nprint(\"train data shape:\\t\", df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:11.296636Z","iopub.execute_input":"2024-05-14T16:46:11.297001Z","iopub.status.idle":"2024-05-14T16:46:17.57054Z","shell.execute_reply.started":"2024-05-14T16:46:11.296968Z","shell.execute_reply":"2024-05-14T16:46:17.569339Z"},"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"train data shape:\t (1526659, 376)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Test Files Read & Feature Engineering","metadata":{}},{"cell_type":"code","source":"data_store = {\n \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n \"depth_0\": [\n read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n ],\n \"depth_1\": [\n read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n ],\n \"depth_2\": [\n read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:17.57209Z","iopub.execute_input":"2024-05-14T16:46:17.572546Z","iopub.status.idle":"2024-05-14T16:46:17.938217Z","shell.execute_reply.started":"2024-05-14T16:46:17.572497Z","shell.execute_reply":"2024-05-14T16:46:17.937264Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\n\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:17.940997Z","iopub.execute_input":"2024-05-14T16:46:17.941317Z","iopub.status.idle":"2024-05-14T16:46:17.975089Z","shell.execute_reply.started":"2024-05-14T16:46:17.94129Z","shell.execute_reply":"2024-05-14T16:46:17.973969Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"test data shape:\t (10, 375)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# My Functions","metadata":{}},{"cell_type":"code","source":"# override Optuna's default logging to ERROR only\noptuna.logging.set_verbosity(optuna.logging.ERROR)\n\n# define a logging callback that will report on only new challenger parameter configurations if a\n# trial has usurped the state of 'best conditions'\ndef champion_callback(study, frozen_trial):\n \"\"\"\n Logging callback that will report when a new trial iteration improves upon existing\n best trial values.\n\n Note: This callback is not intended for use in distributed computing systems such as Spark\n or Ray due to the micro-batch iterative implementation for distributing trials to a cluster's\n workers or agents.\n The race conditions with file system state management for distributed trials will render\n inconsistent values with this callback.\n \"\"\"\n\n winner = study.user_attrs.get(\"winner\", None)\n\n if study.best_value and winner != study.best_value:\n study.set_user_attr(\"winner\", study.best_value)\n if winner:\n improvement_percent = (abs(winner - study.best_value) / study.best_value) * 100\n print(\n f\"Trial {frozen_trial.number} achieved value: {frozen_trial.value} with \"\n f\"{improvement_percent: .4f}% improvement\"\n )\n else:\n print(f\"Initial trial {frozen_trial.number} achieved value: {frozen_trial.value}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:17.976379Z","iopub.execute_input":"2024-05-14T16:46:17.976745Z","iopub.status.idle":"2024-05-14T16:46:17.986202Z","shell.execute_reply.started":"2024-05-14T16:46:17.976715Z","shell.execute_reply":"2024-05-14T16:46:17.98499Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"def preprocessing_data(X, y, test_size, stratify, random_state=42):\n \"\"\"\n Preprocessing data by applying normalization on numerical columns and one hot encoding on categorical columns.\n\n This function receives X and y data and return X_train, X_test, y_train, y_test.\n\n Parameters:\n - X (pandas DataFrame): independent variables.\n\n - y (pandas Series): target variable.\n\n Returns:\n - X_train (pandas DataFrame): independent variables to be used to train models.\n\n - X_test (pandas DataFrame): independent variables to be used evaluate models.\n\n - y_train (pandas Series): target variable to be use to train models.\n\n - y_test (pandas Series): target variable to be use to evaluate models.\n \"\"\"\n\n X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state, stratify=stratify)\n\n # numerical columns\n numerical_columns = X_train.select_dtypes(include=np.number).columns\n\n X_train_scaled = X_train.copy()\n X_test_scaled = X_test.copy()\n\n scalers = {}\n for column in numerical_columns:\n scaler = RobustScaler()\n X_train_scaled[column] = scaler.fit_transform(X_train[[column]])\n X_test_scaled[column] = scaler.transform(X_test[[column]])\n scalers[column] = scaler\n\n\n # categorical columns\n categorical_columns = X_train.select_dtypes(include=['object']).columns\n\n # Encoding multiple columns. Unfortunately you cannot pass a list here\n # so you need to copy-paste all printed categorical columns.\n transformer = make_column_transformer(\n (OneHotEncoder(sparse=False, handle_unknown='ignore', dtype='int'), categorical_columns),\n verbose_feature_names_out=False\n )\n\n # applying ohe hot encoding on training data\n X_train_transformed = transformer.fit_transform(X_train[categorical_columns])\n # transforming in pandas DataFrame\n X_train_transformed = pd.DataFrame(X_train_transformed, columns=transformer.get_feature_names_out())\n # One-hot encoding removed an index. Let's put it back:\n X_train_transformed.index = X_train.index\n # joining tables\n X_train = pd.concat([X_train, X_train_transformed], axis=1)\n # dropping old categorical columns\n X_train.drop(categorical_columns, axis=1, inplace=True)\n\n # applying ohe hot encoding on testing data\n X_test_transformed = transformer.transform(X_test)\n # transforming in pandas DataFrame\n X_test_transformed = pd.DataFrame(X_test_transformed, columns=transformer.get_feature_names_out())\n # One-hot encoding removed an index. Let's put it back:\n X_test_transformed.index = X_test.index\n # Joining tables\n X_test = pd.concat([X_test, X_test_transformed], axis=1)\n # Dropping old categorical columns\n X_test.drop(categorical_columns, axis=1, inplace=True)\n\n return X_train, X_test, y_train, y_test, scalers, transformer","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:17.987916Z","iopub.execute_input":"2024-05-14T16:46:17.988693Z","iopub.status.idle":"2024-05-14T16:46:18.006243Z","shell.execute_reply.started":"2024-05-14T16:46:17.988651Z","shell.execute_reply":"2024-05-14T16:46:18.005085Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"# creating a personalized confusion matrix\ndef plot_confusion_matrix(y_test, y_pred):\n \"\"\"\n Plot a confusion matrix in a heatmap format for better visualization.\n\n This function receives real y values and predicted y values and create a plot for the confusion matrix.\n\n Parameters:\n - y_test (pandas Series): real values of y (target) variable.\n\n - y_tpred (pandas Series): predicted values of y (target) variable by the model.\n\n Returns:\n - figure: confusion matrix plot figure.\n \"\"\"\n\n matrix = confusion_matrix(y_test, y_pred)\n\n fig, (ax1, ax2) = plt.subplots(1,2, figsize=(8,3))\n fig.suptitle('Confusion Matrix', y=1.1)\n\n sns.heatmap(matrix, annot=True, fmt='d', cmap='Blues', ax=ax1)\n ax1.set_xlabel('Predicted Values')\n ax1.set_ylabel('Real Values')\n\n # criando mapa de calor com valores relativos\n sns.heatmap(matrix / matrix.sum(), annot=True, fmt='.2%', cmap='Blues', ax=ax2)\n ax2.set_xlabel('Predicted Values')\n ax2.set_ylabel('Real Values')\n fig.tight_layout()\n\n return fig","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:18.007484Z","iopub.execute_input":"2024-05-14T16:46:18.007818Z","iopub.status.idle":"2024-05-14T16:46:18.023924Z","shell.execute_reply.started":"2024-05-14T16:46:18.007789Z","shell.execute_reply":"2024-05-14T16:46:18.022766Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"# ROC Curve Plot\ndef plot_roc_curve(model, X_test, y_test):\n \"\"\"\n Plot of the ROC curve.\n\n This function receives the model, X_test and the y_test and returns the ROC curve figure.\n\n Parameters:\n - model: a trained model.\n\n - X_test (pandas DataFrame): pandas test DataFrame.\n\n - y_test (pandas Series): pandas test Series.\n\n Returns:\n - figure: ROC curve figure.\n \"\"\"\n\n y_pred_probs = model.predict_proba(X_test)[:,1]\n fpr, tpr, thresholds = roc_curve(y_test, y_pred_probs)\n logit_roc_auc = roc_auc_score(y_test, y_pred_probs)\n\n fig = plt.figure(figsize=(4,2))\n plt.plot(fpr, tpr, label=f'(área = {round(logit_roc_auc, 2)})')\n plt.plot([0, 1], [0, 1],'r--')\n plt.xlabel('False Positive Rate')\n plt.ylabel('True Positive Rate')\n plt.title('ROC Curve')\n plt.legend(loc=\"lower right\")\n\n return fig","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:18.025235Z","iopub.execute_input":"2024-05-14T16:46:18.02562Z","iopub.status.idle":"2024-05-14T16:46:18.035569Z","shell.execute_reply.started":"2024-05-14T16:46:18.025566Z","shell.execute_reply":"2024-05-14T16:46:18.034643Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n gini_in_time = base[[\"WEEK_NUM\", \"target\", \"score\"]]\\\n .sort_values(\"WEEK_NUM\")\\\n .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n \n x = np.arange(len(gini_in_time))\n y = gini_in_time\n a, b = np.polyfit(x, y, 1)\n y_hat = a*x + b\n residuals = y - y_hat\n res_std = np.std(residuals)\n avg_gini = np.mean(gini_in_time)\n return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:18.037078Z","iopub.execute_input":"2024-05-14T16:46:18.03739Z","iopub.status.idle":"2024-05-14T16:46:18.052369Z","shell.execute_reply.started":"2024-05-14T16:46:18.037364Z","shell.execute_reply":"2024-05-14T16:46:18.051382Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"markdown","source":"### Feature Elimination","metadata":{}},{"cell_type":"code","source":"#df_train = df_train.pipe(Pipeline.filter_cols)\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\n\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:18.056049Z","iopub.execute_input":"2024-05-14T16:46:18.05638Z","iopub.status.idle":"2024-05-14T16:46:18.068286Z","shell.execute_reply.started":"2024-05-14T16:46:18.05634Z","shell.execute_reply":"2024-05-14T16:46:18.06713Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"train data shape:\t (1526659, 376)\ntest data shape:\t (10, 375)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Pandas Conversion","metadata":{}},{"cell_type":"code","source":"df_train, cat_cols = to_pandas(df_train)\ndf_test, cat_cols = to_pandas(df_test, cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:18.069928Z","iopub.execute_input":"2024-05-14T16:46:18.070341Z","iopub.status.idle":"2024-05-14T16:46:42.605689Z","shell.execute_reply.started":"2024-05-14T16:46:18.070303Z","shell.execute_reply":"2024-05-14T16:46:42.604633Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"feature_definitions = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:42.607518Z","iopub.execute_input":"2024-05-14T16:46:42.607876Z","iopub.status.idle":"2024-05-14T16:46:42.618592Z","shell.execute_reply.started":"2024-05-14T16:46:42.607843Z","shell.execute_reply":"2024-05-14T16:46:42.617173Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"markdown","source":"### Garbage Collection","metadata":{}},{"cell_type":"code","source":"del data_store\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:42.620385Z","iopub.execute_input":"2024-05-14T16:46:42.620809Z","iopub.status.idle":"2024-05-14T16:46:42.774718Z","shell.execute_reply.started":"2024-05-14T16:46:42.620771Z","shell.execute_reply":"2024-05-14T16:46:42.773446Z"},"trusted":true},"execution_count":21,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"0"},"metadata":{}}]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"print(\"Train is duplicated:\\t\", df_train[\"case_id\"].duplicated().any())\nprint(\"Train Week Range:\\t\", (df_train[\"WEEK_NUM\"].min(), df_train[\"WEEK_NUM\"].max()))\n\nprint()\n\nprint(\"Test is duplicated:\\t\", df_test[\"case_id\"].duplicated().any())\nprint(\"Test Week Range:\\t\", (df_test[\"WEEK_NUM\"].min(), df_test[\"WEEK_NUM\"].max()))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:42.777454Z","iopub.execute_input":"2024-05-14T16:46:42.777986Z","iopub.status.idle":"2024-05-14T16:46:42.831335Z","shell.execute_reply.started":"2024-05-14T16:46:42.777943Z","shell.execute_reply":"2024-05-14T16:46:42.830129Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"Train is duplicated:\t False\nTrain Week Range:\t (0, 91)\n\nTest is duplicated:\t False\nTest Week Range:\t (100, 100)\n","output_type":"stream"}]},{"cell_type":"code","source":"#Feature non-null proportion\ndf_missing = (df_train.isna().sum() / df_train.shape[0]).reset_index()\ndf_missing.columns = ['var','percent']\ndf_missing.sort_values('percent', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:42.832918Z","iopub.execute_input":"2024-05-14T16:46:42.833326Z","iopub.status.idle":"2024-05-14T16:46:43.64682Z","shell.execute_reply.started":"2024-05-14T16:46:42.833288Z","shell.execute_reply":"2024-05-14T16:46:43.645478Z"},"trusted":true},"execution_count":23,"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":" var percent\n85 clientscnt_136L 0.999727\n317 max_periodicityofpmts_997L 0.999161\n140 lastrepayingdate_696D 0.998443\n132 lastotherinc_902A 0.997998\n133 lastotherlnsexpense_631A 0.997997\n.. ... ...\n84 clientscnt_1130L 0.000000\n83 clientscnt_1071L 0.000000\n82 clientscnt_1022L 0.000000\n81 clientscnt_100L 0.000000\n90 clientscnt_493L 0.000000\n\n[376 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
varpercent
85clientscnt_136L0.999727
317max_periodicityofpmts_997L0.999161
140lastrepayingdate_696D0.998443
132lastotherinc_902A0.997998
133lastotherlnsexpense_631A0.997997
.........
84clientscnt_1130L0.000000
83clientscnt_1071L0.000000
82clientscnt_1022L0.000000
81clientscnt_100L0.000000
90clientscnt_493L0.000000
\n

376 rows × 2 columns

\n
"},"metadata":{}}]},{"cell_type":"markdown","source":"## Categorical Features","metadata":{}},{"cell_type":"markdown","source":"### Select relevant features","metadata":{}},{"cell_type":"code","source":"categorical_cols = df_train.select_dtypes(exclude=np.number).columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:43.648656Z","iopub.execute_input":"2024-05-14T16:46:43.649054Z","iopub.status.idle":"2024-05-14T16:46:43.693894Z","shell.execute_reply.started":"2024-05-14T16:46:43.649008Z","shell.execute_reply":"2024-05-14T16:46:43.692623Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"markdown","source":"significance = dict with all significance values","metadata":{}},{"cell_type":"code","source":"categorical_cols_1 = categorical_cols[:100]\n#categorical_cols[100:200]","metadata":{"execution":{"iopub.status.busy":"2024-05-14T16:46:43.695692Z","iopub.execute_input":"2024-05-14T16:46:43.696032Z","iopub.status.idle":"2024-05-14T16:46:43.700568Z","shell.execute_reply.started":"2024-05-14T16:46:43.696003Z","shell.execute_reply":"2024-05-14T16:46:43.699377Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"categorical_cols_1.append('target')\ncategorical_cols_1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:07:11.898107Z","iopub.execute_input":"2024-05-14T18:07:11.898868Z","iopub.status.idle":"2024-05-14T18:07:11.928874Z","shell.execute_reply.started":"2024-05-14T18:07:11.89883Z","shell.execute_reply":"2024-05-14T18:07:11.927271Z"},"trusted":true},"execution_count":32,"outputs":[{"execution_count":32,"output_type":"execute_result","data":{"text/plain":" case_id WEEK_NUM target month_decision weekday_decision \\\n0 0 0 0 1 4 \n1 1 0 0 1 4 \n2 2 0 0 1 5 \n3 3 0 0 1 4 \n4 4 0 1 1 5 \n\n assignmentdate_238D assignmentdate_4527235D assignmentdate_4955616D \\\n0 NaN NaN NaN \n1 NaN NaN NaN \n2 NaN NaN NaN \n3 NaN NaN NaN \n4 NaN NaN NaN \n\n birthdate_574D contractssum_5085716L ... \\\n0 NaN NaN ... \n1 NaN NaN ... \n2 NaN NaN ... \n3 NaN NaN ... \n4 NaN NaN ... \n\n max_last180dayaveragebalance_704A max_last180dayturnover_1134A \\\n0 NaN NaN \n1 NaN NaN \n2 NaN NaN \n3 NaN NaN \n4 NaN NaN \n\n max_last30dayturnover_651A max_openingdate_857D max_num_group1_10 \\\n0 NaN NaN NaN \n1 NaN NaN NaN \n2 NaN NaN NaN \n3 NaN NaN NaN \n4 NaN NaN NaN \n\n max_pmts_dpdvalue_108P max_pmts_pmtsoverdue_635A max_pmts_date_1107D \\\n0 NaN NaN NaN \n1 NaN NaN NaN \n2 NaN NaN NaN \n3 NaN NaN NaN \n4 NaN NaN NaN \n\n max_num_group1_11 max_num_group2 \n0 NaN NaN \n1 NaN NaN \n2 NaN NaN \n3 NaN NaN \n4 NaN NaN \n\n[5 rows x 376 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
case_idWEEK_NUMtargetmonth_decisionweekday_decisionassignmentdate_238Dassignmentdate_4527235Dassignmentdate_4955616Dbirthdate_574Dcontractssum_5085716L...max_last180dayaveragebalance_704Amax_last180dayturnover_1134Amax_last30dayturnover_651Amax_openingdate_857Dmax_num_group1_10max_pmts_dpdvalue_108Pmax_pmts_pmtsoverdue_635Amax_pmts_date_1107Dmax_num_group1_11max_num_group2
000014NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
110014NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
220015NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
330014NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
440115NaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n

5 rows × 376 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"df_train_1 = df_train[categorical_cols_1]\ndf_train_1.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:06:59.467905Z","iopub.execute_input":"2024-05-14T18:06:59.468288Z","iopub.status.idle":"2024-05-14T18:06:59.56487Z","shell.execute_reply.started":"2024-05-14T18:06:59.468257Z","shell.execute_reply":"2024-05-14T18:06:59.563616Z"},"trusted":true},"execution_count":31,"outputs":[{"execution_count":31,"output_type":"execute_result","data":{"text/plain":" description_5085714M education_1103M education_88M maritalst_385M \\\n0 NaN NaN NaN NaN \n1 NaN NaN NaN NaN \n2 NaN NaN NaN NaN \n3 NaN NaN NaN NaN \n4 NaN NaN NaN NaN \n\n maritalst_893M requesttype_4525192L riskassesment_302T bankacctype_710L \\\n0 NaN NaN NaN NaN \n1 NaN NaN NaN NaN \n2 NaN NaN NaN NaN \n3 NaN NaN NaN NaN \n4 NaN NaN NaN NaN \n\n cardtype_51L credtype_322L ... max_maritalst_703L \\\n0 NaN CAL ... NaN \n1 NaN CAL ... NaN \n2 NaN CAL ... NaN \n3 NaN CAL ... NaN \n4 NaN CAL ... NaN \n\n max_relationshiptoclient_415T max_relationshiptoclient_642T \\\n0 SPOUSE SPOUSE \n1 SIBLING SIBLING \n2 SPOUSE SPOUSE \n3 SPOUSE SPOUSE \n4 SIBLING SIBLING \n\n max_remitter_829L max_role_1084L max_role_993L max_safeguarantyflag_411L \\\n0 False PE NaN True \n1 False PE NaN True \n2 False PE NaN True \n3 False PE NaN True \n4 False PE NaN True \n\n max_sex_738L max_type_25L target \n0 F PRIMARY_MOBILE 0 \n1 M PRIMARY_MOBILE 0 \n2 F PRIMARY_MOBILE 0 \n3 F PRIMARY_MOBILE 0 \n4 F PRIMARY_MOBILE 1 \n\n[5 rows x 86 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
description_5085714Meducation_1103Meducation_88Mmaritalst_385Mmaritalst_893Mrequesttype_4525192Lriskassesment_302Tbankacctype_710Lcardtype_51Lcredtype_322L...max_maritalst_703Lmax_relationshiptoclient_415Tmax_relationshiptoclient_642Tmax_remitter_829Lmax_role_1084Lmax_role_993Lmax_safeguarantyflag_411Lmax_sex_738Lmax_type_25Ltarget
0NaNNaNNaNNaNNaNNaNNaNNaNNaNCAL...NaNSPOUSESPOUSEFalsePENaNTrueFPRIMARY_MOBILE0
1NaNNaNNaNNaNNaNNaNNaNNaNNaNCAL...NaNSIBLINGSIBLINGFalsePENaNTrueMPRIMARY_MOBILE0
2NaNNaNNaNNaNNaNNaNNaNNaNNaNCAL...NaNSPOUSESPOUSEFalsePENaNTrueFPRIMARY_MOBILE0
3NaNNaNNaNNaNNaNNaNNaNNaNNaNCAL...NaNSPOUSESPOUSEFalsePENaNTrueFPRIMARY_MOBILE0
4NaNNaNNaNNaNNaNNaNNaNNaNNaNCAL...NaNSIBLINGSIBLINGFalsePENaNTrueFPRIMARY_MOBILE1
\n

5 rows × 86 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"significance = {}\nfor i in categorical_cols_1: \n significance[i] = {}\n \n for j in df_train[~df_train[i].isna()][i].unique():\n #null_count\n df_filtered_nulls = df_train[['target',i]][df_train[i].isna()]\n null_count = round((len(df_filtered_nulls) / len(df_train))*100,1)\n significance[i]['null'] = {}\n significance[i]['null']['count'] = {null_count}\n \n #non-null values\n df_filtered_category = df_train[['target',i]][df_train[i] == j]\n \n count = round((len(df_filtered_category) / len(df_train)) * 100,1)\n significance[i][j] = {}\n significance[i][j]['count'] = {count}\n \n target_dist = round((len(df_filtered_category[df_filtered_category['target'] == 1]) / len(df_train))*100,1)\n significance[i][j]['target_dist'] = {target_dist} ","metadata":{"execution":{"iopub.status.busy":"2024-05-14T18:07:48.347248Z","iopub.execute_input":"2024-05-14T18:07:48.347821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Threshold = 1 percent","metadata":{}},{"cell_type":"markdown","source":"filtered_significants = dict with all significant values and blank for non significant values \\\nsignificant_cols = list with filtered important columns","metadata":{}},{"cell_type":"code","source":"#define percentage threshold\nthreshold = 1\n\nfiltered_significants = {}\nfor key, value in significance.items():\n filtered_significants[key] = {}\n for key1 in value.keys(): \n target_value = value[key1].get('target_dist')\n count = value[key1].get('count')\n if target_value is None:\n pass\n elif list(target_value)[0] > threshold:\n #print(key,key1,list(target_value)[0])\n filtered_significants[key][key1] = {'target_dist':list(target_value)[0],\n 'share':list(count)[0]}\n else:\n pass\n\nfiltered_significants","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Threshold = 3 percent","metadata":{}},{"cell_type":"markdown","source":"filtered_significants = dict filtered significant features \\\nsignificant_cols = list with filtered important columns","metadata":{}},{"cell_type":"code","source":"#define significance threshold in percentage\nthreshold = 3\n\nsignificant_cols = set()\nfiltered_significants = {}\nfor key, value in significance.items():\n filtered_significants[key] = {}\n for key1 in value.keys(): \n target_value = value[key1].get('target_dist')\n count = value[key1].get('count')\n if target_value is None:\n pass\n elif list(target_value)[0] > threshold:\n filtered_significants[key][key1] = {'target_dist':list(target_value)[0],\n 'share':list(count)[0]}\n significant_cols.add(key)\n else:\n pass\nsignificant_features = list(significant_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"significant_cols = set()\nfiltered_significants = {}\nfor key, value in significance.items():\n filtered_significants[key] = {}\n if key in significant_features:\n for key1 in value.keys(): \n target_value = value[key1].get('target_dist')\n count = value[key1].get('count')\n if target_value is None:\n pass\n elif list(target_value)[0] > threshold:\n filtered_significants[key][key1] = {'target_dist':list(target_value)[0],\n 'share':list(count)[0]}\n significant_cols.add(key)\n else:\n pass\n else:\n pass\n \nsignificant_features = list(significant_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"significant_features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Turning it into a function","metadata":{}},{"cell_type":"code","source":"def filter_significants(dic,threshold):\n #define significance threshold in percentage\n filtered_significants = {}\n for key, value in dic.items():\n filtered_significants[key] = {}\n for key1 in value.keys(): \n target_value = value[key1].get('target_dist')\n count = value[key1].get('count')\n if target_value is None:\n pass\n elif list(target_value)[0] > threshold:\n #print(key,key1,list(target_value)[0])\n filtered_significants[key][key1] = {'target_dist':list(target_value)[0],\n 'share':list(count)[0]}\n else:\n pass\n print(filtered_significants)\n return filtered_significants\nfiltered_significants_3 = filtered_significants(significance,3)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Domain Analysis","metadata":{}},{"cell_type":"code","source":"#Selected domains\ncol_lis = ['Variable','equalitydataagreement_891L','equalityempfrom_62L','credacc_actualbalance_314A','credacc_maxhisbal_375A','credacc_minhisbal_90A','maxoutstandbalancel12m_4187113A','currdebt_22A','currdebt_94A','currdebtcredtyperange_828A','totaldebt_9A','education_1103M','education_1138M','education_88M','education_927M','housetype_905L','housingtype_772L','commnoinclast6m_3546845L','incometype_1044T','isreference_387L','maininc_215A','mainoccupationinc_384A','eir_270L','interestrategrace_34L','dtlastpmt_581D','dtlastpmtallstes_3545839D','dtlastpmtallstes_4499206D','maxlnamtstart6m_4525199A','sumoutstandtotal_3546847A','sumoutstandtotalest_4493215A','overdueamount_31A','overdueamount_659A','overdueamountmax_155A','overdueamountmax_35A','overdueamountmax_950A','overdueamountmax2_14A','overdueamountmax2_398A','overdueamountmax2date_1002D','overdueamountmax2date_1142D','overdueamountmaxdatemonth_284T','overdueamountmaxdatemonth_365T','overdueamountmaxdatemonth_494T','overdueamountmaxdateyear_2T','overdueamountmaxdateyear_432T','overdueamountmaxdateyear_994T','avgdbddpdlast24m_3658932P','avgdbddpdlast3m_4187120P','avgdbdtollast24m_4525197P','avgdpdtolclosure24_3658938P','avginstallast24m_3658937A','avglnamtstart24m_4525187A','avgmaxdpdlast9m_3716943P','avgoutstandbalancel6m_4187114A','avgpmtlast12m_4525200A','cntincpaycont9m_3716944L','cntpmts24_3658933L','downpmt_116A','downpmt_134A','lastrepayingdate_696D','maxpmtlast3m_4525190A','monthlyinstlamount_674A','numincomingpmts_3546848L','numpmtchanneldd_318L','payvacationpostpone_4187118D','pmtmethod_731M','pmtnum_254L','pmtnum_8L','pmtnumpending_403L','pmts_date_1107D','pmts_dpd_1073P','pmts_dpd_303P','pmts_dpdvalue_108P','pmts_month_158T','pmts_month_706T','pmts_overdue_1140A','pmts_overdue_1152A','pmts_pmtsoverdue_635A','pmts_year_1139T','pmts_year_507T','pmtscount_423L','totalsettled_863A','formonth_118L','foryear_618L','lastrejectreason_759M','lastrejectreasonclient_4145040M','rejectreason_755M','rejectreasonclient_4145042M','role_1084L','role_993L']\ncat_lis = ['category','alert flag','alert flag','balance','balance','balance','balance','current debt','current debt','current debt','debt','education level','education level','education level','education level','house','house','income info','income info','income info','income info','income info','interest','interest','last payment','last payment','last payment','loan','outstanding amount','outstanding amount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','overdue ammount','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','payments','rejection','rejection','rejection','rejection','rejection','rejection','role','role']\ndf_cats = pd.DataFrame({'var':col_lis,'category':cat_lis}).drop(0)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filter_lis = col_lis.copy()\nfilter_lis += ['target']\nfilter_lis.insert(0,'case_id')\ndf_selected_var = df_train.filter(items=filter_lis)#.copy()\ndf_selected_var.columns#['formonth_118L']","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_cols = len(df_selected_var.columns)\ntarget_dist = len(df_selected_var[df_selected_var['target'] == 1]) / len(df_selected_var) \nprint('number of cols:' , n_cols)\nprint('target default %:', int(target_dist*100))","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = df_selected_var.select_dtypes(exclude=np.number).columns.tolist()\nnumerical_cols = df_selected_var.select_dtypes(include=np.number).columns.tolist()\ncategorical_cols","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[columns_to_maintain]\ndf_test = df_test[columns_to_maintain_test]","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = df_train.select_dtypes(exclude=np.number).columns.tolist()\nnumerical_cols = df_train.select_dtypes(include=np.number).columns.tolist()","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rejection","metadata":{}},{"cell_type":"code","source":"#'formonth_118L','foryear_618L', 'rejectreason_755M', 'rejectreasonclient_4145042M'\ncols = ['lastrejectreason_759M','lastrejectreasonclient_4145040M']\ncols.insert(0,'target')\n#select cols\ndf_eda = df_selected_var[cols]\n\n#size missing values\ndf_missing = (df_eda.isna().sum() / df_train.shape[0]).reset_index()\ndf_missing.columns = ['var','percent']\ndf_missing.sort_values('percent', ascending=False)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = df_eda.select_dtypes(exclude=np.number).columns.tolist()\nnumerical_cols = df_eda.drop('target',axis=1).select_dtypes(include=np.number).columns.tolist()\nprint('categorical_cols', categorical_cols)\nprint('numerical_cols', numerical_cols)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Alert Flags","metadata":{}},{"cell_type":"markdown","source":"## Education Level and Role","metadata":{}},{"cell_type":"code","source":"#['education_1138M', 'education_927M', 'role_1084L', 'role_993L']\ncols = ['education_88M']\n#'formonth_118L','foryear_618L', 'rejectreason_755M', 'rejectreasonclient_4145042M'\ncols.insert(0,'target')\n#select cols\ndf_eda = df_selected_var[cols]\n\n#size missing values\ndf_missing = (df_eda.isna().sum() / df_train.shape[0]).reset_index()\ndf_missing.columns = ['var','percent']\ndf_missing.sort_values('percent', ascending=False)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = df_eda.select_dtypes(exclude=np.number).columns.tolist()\nnumerical_cols = df_eda.drop('target',axis=1).select_dtypes(include=np.number).columns.tolist()\nprint('categorical_cols', categorical_cols)\nprint('numerical_cols', numerical_cols)","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Numerical Features","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}