{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":50160,"databundleVersionId":7921029},{"sourceType":"datasetVersion","sourceId":8161904,"datasetId":4589852,"databundleVersionId":8283730}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.feature_selection import VarianceThreshold\nfrom sklearn.metrics import roc_auc_score \nimport pickle\nimport matplotlib.pyplot as plt\nimport os \nfrom gc import collect\nfrom os import path, walk, getpid\nfrom psutil import Process\nimport ctypes;\nlibc = ctypes.CDLL(\"libc.so.6\");","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-19T08:03:43.808484Z","iopub.execute_input":"2024-04-19T08:03:43.808914Z","iopub.status.idle":"2024-04-19T08:03:49.472550Z","shell.execute_reply.started":"2024-04-19T08:03:43.808877Z","shell.execute_reply":"2024-04-19T08:03:49.471283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    data_path = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n    train_base_path = \"/kaggle/input/folded-base-train/train_base_week_folds.csv\"\n    train_path         = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train\";\n    test_path          = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test\";\n    covid_date = \"2020-02-29\"","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:03:49.475479Z","iopub.execute_input":"2024-04-19T08:03:49.476209Z","iopub.status.idle":"2024-04-19T08:03:49.482489Z","shell.execute_reply.started":"2024-04-19T08:03:49.476159Z","shell.execute_reply":"2024-04-19T08:03:49.481177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Utilities:\n    @staticmethod\n    def set_table_dtypes(df: pl.DataFrame) -> pl.DataFrame:\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64));\n            elif len(col) == 0:\n                df = df.drop(col)\n            elif col in [\"date_decision\"] or col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date));\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64));\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String));\n\n        return df;\n    @staticmethod \n    def read_file(path):\n        df = pl.read_csv(path).pipe(Utilities.set_table_dtypes)\n        return df \n    @staticmethod\n    def read_files(regex_path):\n        chunks = []\n        for path in glob(str(regex_path)):\n            chunks.append(Utilities.read_file(path))\n        df = pl.concat(chunks, how='vertical_relaxed')\n        return df\n    @staticmethod\n    def CleanMemory():\n        \"This method cleans the memory off unused objects and displays the cleaned state RAM usage\";\n\n        collect();\n        libc.malloc_trim(0);\n        pid        = getpid();\n        py         = Process(pid);\n        memory_use = py.memory_info()[0] / 2. ** 30;\n        return f\"RAM usage = {memory_use :.4} GB\";","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:03:49.484297Z","iopub.execute_input":"2024-04-19T08:03:49.485005Z","iopub.status.idle":"2024-04-19T08:03:49.506037Z","shell.execute_reply.started":"2024-04-19T08:03:49.484964Z","shell.execute_reply":"2024-04-19T08:03:49.504692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import static internal \nstatic = Utilities.read_files(f'{Config.train_path}/train_static_0_*.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:03:49.509430Z","iopub.execute_input":"2024-04-19T08:03:49.509812Z","iopub.status.idle":"2024-04-19T08:04:05.995266Z","shell.execute_reply.started":"2024-04-19T08:03:49.509780Z","shell.execute_reply":"2024-04-19T08:04:05.993926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import base \nbase = Utilities.read_files(f'{Config.train_path}/train_base.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:05.999209Z","iopub.execute_input":"2024-04-19T08:04:05.999807Z","iopub.status.idle":"2024-04-19T08:04:06.667010Z","shell.execute_reply.started":"2024-04-19T08:04:05.999773Z","shell.execute_reply":"2024-04-19T08:04:06.665811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_def = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:06.668554Z","iopub.execute_input":"2024-04-19T08:04:06.668927Z","iopub.status.idle":"2024-04-19T08:04:06.687260Z","shell.execute_reply.started":"2024-04-19T08:04:06.668892Z","shell.execute_reply":"2024-04-19T08:04:06.686373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"static = static.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:06.688661Z","iopub.execute_input":"2024-04-19T08:04:06.689909Z","iopub.status.idle":"2024-04-19T08:04:09.781477Z","shell.execute_reply.started":"2024-04-19T08:04:06.689832Z","shell.execute_reply":"2024-04-19T08:04:09.780155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = base.to_pandas()\nstatic = static.merge(base[['case_id', 'date_decision', 'target']])","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:09.783440Z","iopub.execute_input":"2024-04-19T08:04:09.783769Z","iopub.status.idle":"2024-04-19T08:04:12.033975Z","shell.execute_reply.started":"2024-04-19T08:04:09.783740Z","shell.execute_reply":"2024-04-19T08:04:12.032852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_colwidth', 100)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:12.035586Z","iopub.execute_input":"2024-04-19T08:04:12.035978Z","iopub.status.idle":"2024-04-19T08:04:12.041465Z","shell.execute_reply.started":"2024-04-19T08:04:12.035945Z","shell.execute_reply":"2024-04-19T08:04:12.040026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Date features","metadata":{}},{"cell_type":"code","source":"date_feats = static.select_dtypes(include=['datetime'])","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:12.046802Z","iopub.execute_input":"2024-04-19T08:04:12.047182Z","iopub.status.idle":"2024-04-19T08:04:12.744693Z","shell.execute_reply.started":"2024-04-19T08:04:12.047152Z","shell.execute_reply":"2024-04-19T08:04:12.743603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_nulls = date_feats.isna().sum() / len(static)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:12.746047Z","iopub.execute_input":"2024-04-19T08:04:12.746605Z","iopub.status.idle":"2024-04-19T08:04:12.795585Z","shell.execute_reply.started":"2024-04-19T08:04:12.746575Z","shell.execute_reply":"2024-04-19T08:04:12.794622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables = date_nulls.rename('null_%').reset_index().rename(columns={'index':'feat'})","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:12.796749Z","iopub.execute_input":"2024-04-19T08:04:12.797542Z","iopub.status.idle":"2024-04-19T08:04:12.805436Z","shell.execute_reply.started":"2024-04-19T08:04:12.797509Z","shell.execute_reply":"2024-04-19T08:04:12.804523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables.set_index('feat', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:12.807011Z","iopub.execute_input":"2024-04-19T08:04:12.807884Z","iopub.status.idle":"2024-04-19T08:04:12.823505Z","shell.execute_reply.started":"2024-04-19T08:04:12.807840Z","shell.execute_reply":"2024-04-19T08:04:12.822188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables['date_min'] = date_feats.min()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:12.825607Z","iopub.execute_input":"2024-04-19T08:04:12.826420Z","iopub.status.idle":"2024-04-19T08:04:13.039699Z","shell.execute_reply.started":"2024-04-19T08:04:12.826374Z","shell.execute_reply":"2024-04-19T08:04:13.038573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables['date_max'] = date_feats.max()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:13.040994Z","iopub.execute_input":"2024-04-19T08:04:13.041368Z","iopub.status.idle":"2024-04-19T08:04:13.258794Z","shell.execute_reply.started":"2024-04-19T08:04:13.041339Z","shell.execute_reply":"2024-04-19T08:04:13.257319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables['desc'] = feat_def.set_index('Variable')['Description']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:13.260866Z","iopub.execute_input":"2024-04-19T08:04:13.261320Z","iopub.status.idle":"2024-04-19T08:04:13.269938Z","shell.execute_reply.started":"2024-04-19T08:04:13.261274Z","shell.execute_reply":"2024-04-19T08:04:13.268705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:13.273328Z","iopub.execute_input":"2024-04-19T08:04:13.273680Z","iopub.status.idle":"2024-04-19T08:04:13.297743Z","shell.execute_reply.started":"2024-04-19T08:04:13.273650Z","shell.execute_reply":"2024-04-19T08:04:13.296487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in date_feats.columns:\n    date_feats[c] = (date_feats[c] - static['date_decision']).apply(lambda x: x.days)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:04:13.299195Z","iopub.execute_input":"2024-04-19T08:04:13.299644Z","iopub.status.idle":"2024-04-19T08:05:00.642977Z","shell.execute_reply.started":"2024-04-19T08:04:13.299602Z","shell.execute_reply":"2024-04-19T08:05:00.641786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables = pd.concat([date_variables,date_feats.describe().transpose()], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:00.644608Z","iopub.execute_input":"2024-04-19T08:05:00.644969Z","iopub.status.idle":"2024-04-19T08:05:02.338795Z","shell.execute_reply.started":"2024-04-19T08:05:00.644940Z","shell.execute_reply":"2024-04-19T08:05:02.337492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables ","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:02.340458Z","iopub.execute_input":"2024-04-19T08:05:02.340950Z","iopub.status.idle":"2024-04-19T08:05:02.376738Z","shell.execute_reply.started":"2024-04-19T08:05:02.340908Z","shell.execute_reply":"2024-04-19T08:05:02.375503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Amount related features","metadata":{}},{"cell_type":"code","source":"amount_columns = [c for c in static.columns if c[-1] == 'A']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:02.378108Z","iopub.execute_input":"2024-04-19T08:05:02.378446Z","iopub.status.idle":"2024-04-19T08:05:02.383707Z","shell.execute_reply.started":"2024-04-19T08:05:02.378418Z","shell.execute_reply":"2024-04-19T08:05:02.382419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amount_variables = static[amount_columns].describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:02.385209Z","iopub.execute_input":"2024-04-19T08:05:02.385665Z","iopub.status.idle":"2024-04-19T08:05:05.402495Z","shell.execute_reply.started":"2024-04-19T08:05:02.385624Z","shell.execute_reply":"2024-04-19T08:05:05.401294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amount_variables['desc'] = feat_def.set_index('Variable')['Description']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:05.403950Z","iopub.execute_input":"2024-04-19T08:05:05.404406Z","iopub.status.idle":"2024-04-19T08:05:05.412187Z","shell.execute_reply.started":"2024-04-19T08:05:05.404366Z","shell.execute_reply":"2024-04-19T08:05:05.410929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amount_variables['null_%'] = 1- amount_variables['count'] / len(static)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:05.414143Z","iopub.execute_input":"2024-04-19T08:05:05.414623Z","iopub.status.idle":"2024-04-19T08:05:05.424199Z","shell.execute_reply.started":"2024-04-19T08:05:05.414580Z","shell.execute_reply":"2024-04-19T08:05:05.423081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amount_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:05.425930Z","iopub.execute_input":"2024-04-19T08:05:05.426335Z","iopub.status.idle":"2024-04-19T08:05:05.471914Z","shell.execute_reply.started":"2024-04-19T08:05:05.426300Z","shell.execute_reply":"2024-04-19T08:05:05.470488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked (Categorical) features","metadata":{}},{"cell_type":"code","source":"missing_or_null = 'a55475b1'","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:05.473606Z","iopub.execute_input":"2024-04-19T08:05:05.474084Z","iopub.status.idle":"2024-04-19T08:05:05.480451Z","shell.execute_reply.started":"2024-04-19T08:05:05.474038Z","shell.execute_reply":"2024-04-19T08:05:05.479256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masked_columns = [c for c in static.columns if c[-1] == 'M']\nmasked_features = static[masked_columns]","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:05.481806Z","iopub.execute_input":"2024-04-19T08:05:05.482169Z","iopub.status.idle":"2024-04-19T08:05:05.646157Z","shell.execute_reply.started":"2024-04-19T08:05:05.482139Z","shell.execute_reply":"2024-04-19T08:05:05.645047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masked_variables = masked_features.describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:05.656184Z","iopub.execute_input":"2024-04-19T08:05:05.656617Z","iopub.status.idle":"2024-04-19T08:05:08.626531Z","shell.execute_reply.started":"2024-04-19T08:05:05.656585Z","shell.execute_reply":"2024-04-19T08:05:08.625206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(masked_features['lastapprcommoditycat_1041M'] == missing_or_null).sum()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:08.628391Z","iopub.execute_input":"2024-04-19T08:05:08.629088Z","iopub.status.idle":"2024-04-19T08:05:08.924401Z","shell.execute_reply.started":"2024-04-19T08:05:08.629057Z","shell.execute_reply":"2024-04-19T08:05:08.923132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in masked_features.columns:\n    null_count = (masked_features[c] == missing_or_null).sum()\n    masked_variables.loc[c, 'count'] -= null_count ","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:08.925593Z","iopub.execute_input":"2024-04-19T08:05:08.926140Z","iopub.status.idle":"2024-04-19T08:05:11.293418Z","shell.execute_reply.started":"2024-04-19T08:05:08.926107Z","shell.execute_reply":"2024-04-19T08:05:11.292294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masked_variables['desc'] = feat_def.set_index('Variable')['Description']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:11.294762Z","iopub.execute_input":"2024-04-19T08:05:11.295372Z","iopub.status.idle":"2024-04-19T08:05:11.301941Z","shell.execute_reply.started":"2024-04-19T08:05:11.295341Z","shell.execute_reply":"2024-04-19T08:05:11.300828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masked_variables['null_%'] = 1- masked_variables['count'] / len(static)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:11.303474Z","iopub.execute_input":"2024-04-19T08:05:11.303810Z","iopub.status.idle":"2024-04-19T08:05:11.317333Z","shell.execute_reply.started":"2024-04-19T08:05:11.303781Z","shell.execute_reply":"2024-04-19T08:05:11.316003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"masked_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:11.318862Z","iopub.execute_input":"2024-04-19T08:05:11.319397Z","iopub.status.idle":"2024-04-19T08:05:11.341496Z","shell.execute_reply.started":"2024-04-19T08:05:11.319355Z","shell.execute_reply":"2024-04-19T08:05:11.340003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature importance from previous training ","metadata":{}},{"cell_type":"code","source":"feat_importances = None\nfeat_names = None\nfor i in range(1, 6):\n    with open(f'/kaggle/input/hc-models/lgb_model_E1_V2_{i}.pkl', 'rb') as f:\n        gbm = pickle.load(f)\n    if i == 1:\n        feat_importances = gbm.feature_importance()\n        feat_names = gbm.feature_name()\n    else:\n        feat_importances += gbm.feature_importance()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:34.754445Z","iopub.execute_input":"2024-04-19T08:05:34.755665Z","iopub.status.idle":"2024-04-19T08:05:34.970940Z","shell.execute_reply.started":"2024-04-19T08:05:34.755618Z","shell.execute_reply":"2024-04-19T08:05:34.967119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = pd.DataFrame(columns=['name', 'importance'], data={'name':feat_names, 'importance':feat_importances})","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:37.956140Z","iopub.execute_input":"2024-04-19T08:05:37.956550Z","iopub.status.idle":"2024-04-19T08:05:37.963853Z","shell.execute_reply.started":"2024-04-19T08:05:37.956520Z","shell.execute_reply":"2024-04-19T08:05:37.962806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = fi.sort_values('importance')","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:38.278273Z","iopub.execute_input":"2024-04-19T08:05:38.278972Z","iopub.status.idle":"2024-04-19T08:05:38.285647Z","shell.execute_reply.started":"2024-04-19T08:05:38.278919Z","shell.execute_reply":"2024-04-19T08:05:38.284178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi['contribution'] = (fi['importance'] / fi['importance'].sum()) * 100","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:42.602007Z","iopub.execute_input":"2024-04-19T08:05:42.602440Z","iopub.status.idle":"2024-04-19T08:05:42.612810Z","shell.execute_reply.started":"2024-04-19T08:05:42.602407Z","shell.execute_reply":"2024-04-19T08:05:42.611638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi['contribution'] = fi['contribution'].round(2)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:42.857381Z","iopub.execute_input":"2024-04-19T08:05:42.857788Z","iopub.status.idle":"2024-04-19T08:05:42.863206Z","shell.execute_reply.started":"2024-04-19T08:05:42.857760Z","shell.execute_reply":"2024-04-19T08:05:42.862268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi['mean_imp'] = fi['contribution'].mean().round(2)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:43.056167Z","iopub.execute_input":"2024-04-19T08:05:43.057026Z","iopub.status.idle":"2024-04-19T08:05:43.064282Z","shell.execute_reply.started":"2024-04-19T08:05:43.056984Z","shell.execute_reply":"2024-04-19T08:05:43.062786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi.tail(20)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:43.514099Z","iopub.execute_input":"2024-04-19T08:05:43.514925Z","iopub.status.idle":"2024-04-19T08:05:43.537320Z","shell.execute_reply.started":"2024-04-19T08:05:43.514882Z","shell.execute_reply":"2024-04-19T08:05:43.536110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DPD or days past due features. Almost like date variables.","metadata":{}},{"cell_type":"code","source":"dpd_columns = [c for c in static.columns if c[-1] == 'P']\ndpd_variables = static[dpd_columns]","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:48.948713Z","iopub.execute_input":"2024-04-19T08:05:48.949103Z","iopub.status.idle":"2024-04-19T08:05:49.309216Z","shell.execute_reply.started":"2024-04-19T08:05:48.949073Z","shell.execute_reply":"2024-04-19T08:05:49.308072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dpd_variables = dpd_variables.describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:49.311655Z","iopub.execute_input":"2024-04-19T08:05:49.312154Z","iopub.status.idle":"2024-04-19T08:05:51.545160Z","shell.execute_reply.started":"2024-04-19T08:05:49.312109Z","shell.execute_reply":"2024-04-19T08:05:51.544122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dpd_variables['null_%'] = 1 - dpd_variables['count'] / len(static)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:51.547731Z","iopub.execute_input":"2024-04-19T08:05:51.548214Z","iopub.status.idle":"2024-04-19T08:05:51.554223Z","shell.execute_reply.started":"2024-04-19T08:05:51.548175Z","shell.execute_reply":"2024-04-19T08:05:51.553110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dpd_variables['desc'] = feat_def.set_index('Variable')['Description']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:51.555665Z","iopub.execute_input":"2024-04-19T08:05:51.556601Z","iopub.status.idle":"2024-04-19T08:05:51.570003Z","shell.execute_reply.started":"2024-04-19T08:05:51.556560Z","shell.execute_reply":"2024-04-19T08:05:51.569119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dpd_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:52.098102Z","iopub.execute_input":"2024-04-19T08:05:52.099514Z","iopub.status.idle":"2024-04-19T08:05:52.136517Z","shell.execute_reply.started":"2024-04-19T08:05:52.099349Z","shell.execute_reply":"2024-04-19T08:05:52.135082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Other columns","metadata":{}},{"cell_type":"code","source":"other_columns = [c for c in static.columns if c[-1] in ('T', 'L')]\nother_variables = static[other_columns]","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:52.569370Z","iopub.execute_input":"2024-04-19T08:05:52.570260Z","iopub.status.idle":"2024-04-19T08:05:54.070213Z","shell.execute_reply.started":"2024-04-19T08:05:52.570210Z","shell.execute_reply":"2024-04-19T08:05:54.069106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_num_variables = other_variables.select_dtypes(include=['float', 'int']).describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:54.071997Z","iopub.execute_input":"2024-04-19T08:05:54.072416Z","iopub.status.idle":"2024-04-19T08:05:59.862259Z","shell.execute_reply.started":"2024-04-19T08:05:54.072386Z","shell.execute_reply":"2024-04-19T08:05:59.861173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_num_variables['null_%'] = 1 - other_num_variables['count'] / len(static)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:59.863941Z","iopub.execute_input":"2024-04-19T08:05:59.864308Z","iopub.status.idle":"2024-04-19T08:05:59.871207Z","shell.execute_reply.started":"2024-04-19T08:05:59.864278Z","shell.execute_reply":"2024-04-19T08:05:59.869879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_num_variables = other_num_variables.sort_values('null_%')","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:59.873559Z","iopub.execute_input":"2024-04-19T08:05:59.874277Z","iopub.status.idle":"2024-04-19T08:05:59.893494Z","shell.execute_reply.started":"2024-04-19T08:05:59.874221Z","shell.execute_reply":"2024-04-19T08:05:59.892261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_num_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:59.895123Z","iopub.execute_input":"2024-04-19T08:05:59.896136Z","iopub.status.idle":"2024-04-19T08:05:59.925200Z","shell.execute_reply.started":"2024-04-19T08:05:59.896103Z","shell.execute_reply":"2024-04-19T08:05:59.924292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_cat_variables = other_variables.select_dtypes(exclude=['float', 'int']).describe().transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:05:59.926729Z","iopub.execute_input":"2024-04-19T08:05:59.927345Z","iopub.status.idle":"2024-04-19T08:06:06.682987Z","shell.execute_reply.started":"2024-04-19T08:05:59.927311Z","shell.execute_reply":"2024-04-19T08:06:06.681374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_cat_variables['null_%'] = 1 - other_cat_variables['count'] / len(static)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.684876Z","iopub.execute_input":"2024-04-19T08:06:06.685270Z","iopub.status.idle":"2024-04-19T08:06:06.691783Z","shell.execute_reply.started":"2024-04-19T08:06:06.685223Z","shell.execute_reply":"2024-04-19T08:06:06.690589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_cat_variables['desc'] = feat_def.set_index('Variable')['Description']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.693939Z","iopub.execute_input":"2024-04-19T08:06:06.694743Z","iopub.status.idle":"2024-04-19T08:06:06.706992Z","shell.execute_reply.started":"2024-04-19T08:06:06.694697Z","shell.execute_reply":"2024-04-19T08:06:06.705911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_cat_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.708774Z","iopub.execute_input":"2024-04-19T08:06:06.709514Z","iopub.status.idle":"2024-04-19T08:06:06.735116Z","shell.execute_reply.started":"2024-04-19T08:06:06.709470Z","shell.execute_reply":"2024-04-19T08:06:06.734142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_num_variables['desc'] = feat_def.set_index('Variable')['Description']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.739041Z","iopub.execute_input":"2024-04-19T08:06:06.739793Z","iopub.status.idle":"2024-04-19T08:06:06.750288Z","shell.execute_reply.started":"2024-04-19T08:06:06.739737Z","shell.execute_reply":"2024-04-19T08:06:06.748931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_num_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.752694Z","iopub.execute_input":"2024-04-19T08:06:06.753173Z","iopub.status.idle":"2024-04-19T08:06:06.784822Z","shell.execute_reply.started":"2024-04-19T08:06:06.753127Z","shell.execute_reply":"2024-04-19T08:06:06.783599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_variables","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.786339Z","iopub.execute_input":"2024-04-19T08:06:06.787271Z","iopub.status.idle":"2024-04-19T08:06:06.823760Z","shell.execute_reply.started":"2024-04-19T08:06:06.787220Z","shell.execute_reply":"2024-04-19T08:06:06.822376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"code","source":"features = pd.concat([amount_variables, \n           masked_variables, \n           date_variables, \n           dpd_variables, \n           other_cat_variables, \n           other_num_variables])","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.825245Z","iopub.execute_input":"2024-04-19T08:06:06.825664Z","iopub.status.idle":"2024-04-19T08:06:06.837152Z","shell.execute_reply.started":"2024-04-19T08:06:06.825632Z","shell.execute_reply":"2024-04-19T08:06:06.835814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features['imp'] = fi.set_index('name')['contribution']","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.838928Z","iopub.execute_input":"2024-04-19T08:06:06.839912Z","iopub.status.idle":"2024-04-19T08:06:06.850860Z","shell.execute_reply.started":"2024-04-19T08:06:06.839852Z","shell.execute_reply":"2024-04-19T08:06:06.849405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features['null_%'].hist()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:06.852897Z","iopub.execute_input":"2024-04-19T08:06:06.854247Z","iopub.status.idle":"2024-04-19T08:06:07.204532Z","shell.execute_reply.started":"2024-04-19T08:06:06.854180Z","shell.execute_reply":"2024-04-19T08:06:07.203336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features['unique'].hist()","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:07.206028Z","iopub.execute_input":"2024-04-19T08:06:07.207188Z","iopub.status.idle":"2024-04-19T08:06:07.529296Z","shell.execute_reply.started":"2024-04-19T08:06:07.207145Z","shell.execute_reply":"2024-04-19T08:06:07.528336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features.reset_index().rename(columns={'index':'feat_name'}).to_csv('static_features_eda.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-19T08:06:07.530767Z","iopub.execute_input":"2024-04-19T08:06:07.531384Z","iopub.status.idle":"2024-04-19T08:06:07.546997Z","shell.execute_reply.started":"2024-04-19T08:06:07.531347Z","shell.execute_reply":"2024-04-19T08:06:07.545472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}