{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":7849698,"sourceType":"datasetVersion","datasetId":4603164}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Deepchecks - data integrity and model performance checks\n\nTrain data based on: https://www.kaggle.com/code/greysky/home-credit-baseline","metadata":{}},{"cell_type":"code","source":"!pip install -qq deepchecks","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:36:22.860377Z","iopub.execute_input":"2024-03-15T12:36:22.860736Z","iopub.status.idle":"2024-03-15T12:36:57.303271Z","shell.execute_reply.started":"2024-03-15T12:36:22.860707Z","shell.execute_reply":"2024-03-15T12:36:57.302137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.metrics import make_scorer, roc_auc_score\n\nfrom deepchecks.tabular import Dataset\nfrom deepchecks.tabular.checks import *\n\nfrom imblearn.under_sampling import RandomUnderSampler\n\nimport lightgbm as lgb\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-15T12:36:57.305637Z","iopub.execute_input":"2024-03-15T12:36:57.305966Z","iopub.status.idle":"2024-03-15T12:37:04.900492Z","shell.execute_reply.started":"2024-03-15T12:36:57.305931Z","shell.execute_reply":"2024-03-15T12:37:04.899499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read data from parquet file\ndf_data = pd.read_parquet(\"/kaggle/input/home-credit-train-data/train.parquet\")\n\n# Set multi-level index\ndf_data = df_data.set_index([\"WEEK_NUM\", \"case_id\"])\n\n# Convert bool columns to float type\nobj_cols = list(df_data.select_dtypes(\"object\").columns)\ndf_data[obj_cols] = df_data[obj_cols].astype(float)\n\n# Convert target column to string type\ndf_data[\"target\"] = df_data[\"target\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:37:04.901913Z","iopub.execute_input":"2024-03-15T12:37:04.902335Z","iopub.status.idle":"2024-03-15T12:37:18.418010Z","shell.execute_reply.started":"2024-03-15T12:37:04.902300Z","shell.execute_reply":"2024-03-15T12:37:18.416898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Integrity","metadata":{}},{"cell_type":"code","source":"# Define RandomUnderSampler with a fixed random state\nsampler = RandomUnderSampler(random_state=42)\n\n# Separate features (X) and target variable (y)\nX, y = df_data.drop(columns=[\"target\"]), df_data[\"target\"]\n\n# Undersample the majority class to balance the classes\nX_resampled, y_resampled = sampler.fit_resample(X, y)\n\n# Identify categorical columns in the feature set\ncat_cols = list(X_resampled.select_dtypes(\"category\").columns)\n\n# Create deepchecks dataset\nds_data = Dataset(pd.concat([X_resampled, y_resampled], axis=1), label=\"target\", cat_features=cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:37:18.420390Z","iopub.execute_input":"2024-03-15T12:37:18.420835Z","iopub.status.idle":"2024-03-15T12:37:29.425388Z","shell.execute_reply.started":"2024-03-15T12:37:18.420798Z","shell.execute_reply":"2024-03-15T12:37:29.424150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Is Single Value","metadata":{}},{"cell_type":"code","source":"check = IsSingleValue()\ncheck.add_condition_not_single_value()\n\nresult = check.run(ds_data)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:17:25.848334Z","iopub.execute_input":"2024-03-15T12:17:25.848833Z","iopub.status.idle":"2024-03-15T12:17:28.135083Z","shell.execute_reply.started":"2024-03-15T12:17:25.848799Z","shell.execute_reply":"2024-03-15T12:17:28.132353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Label Correlation","metadata":{}},{"cell_type":"code","source":"check = FeatureLabelCorrelation(ppscore_params={'sample': 20000})\ncheck.add_condition_feature_pps_less_than(0.8)\n\nresult = check.run(dataset=ds_data)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:13:58.054201Z","iopub.execute_input":"2024-03-15T12:13:58.054799Z","iopub.status.idle":"2024-03-15T12:15:15.683229Z","shell.execute_reply.started":"2024-03-15T12:13:58.054757Z","shell.execute_reply":"2024-03-15T12:15:15.680826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Percent of Nulls","metadata":{}},{"cell_type":"code","source":"check = PercentOfNulls()\ncheck.add_condition_percent_of_nulls_not_greater_than(0.95)\n\nresult = check.run(ds_data)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:17:52.350123Z","iopub.execute_input":"2024-03-15T12:17:52.352028Z","iopub.status.idle":"2024-03-15T12:17:54.234836Z","shell.execute_reply.started":"2024-03-15T12:17:52.351947Z","shell.execute_reply":"2024-03-15T12:17:54.233501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Feature Correlation","metadata":{}},{"cell_type":"code","source":"check = FeatureFeatureCorrelation(n_samples=20000)\ncheck.add_condition_max_number_of_pairs_above_threshold(0.9)\n\nresult = check.run(ds_data)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:18:20.394043Z","iopub.execute_input":"2024-03-15T12:18:20.394481Z","iopub.status.idle":"2024-03-15T12:21:41.793515Z","shell.execute_reply.started":"2024-03-15T12:18:20.394447Z","shell.execute_reply":"2024-03-15T12:21:41.790358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Mixed Data Types","metadata":{}},{"cell_type":"code","source":"check = MixedDataTypes()\ncheck.add_condition_rare_type_ratio_not_in_range()\n\nresult = check.run(ds_data)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:37:29.426817Z","iopub.execute_input":"2024-03-15T12:37:29.428358Z","iopub.status.idle":"2024-03-15T12:37:31.789539Z","shell.execute_reply.started":"2024-03-15T12:37:29.428323Z","shell.execute_reply":"2024-03-15T12:37:31.788138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Test Validation","metadata":{}},{"cell_type":"code","source":"weeks = X.index.get_level_values(0)\n\nX_train = X[weeks < 45]\ny_train = y[weeks < 45]\n\nX_test = X[weeks >= 45]\ny_test = y[weeks >= 45]\n\nds_train = Dataset(\n    df=pd.concat([X_train, y_train], axis=1), \n    label=\"target\", \n    cat_features=cat_cols,\n)\n\nds_test = Dataset(\n    df=pd.concat([X_test, y_test], axis=1), \n    label=\"target\", \n    cat_features=cat_cols,\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:38:28.945344Z","iopub.execute_input":"2024-03-15T12:38:28.945778Z","iopub.status.idle":"2024-03-15T12:38:44.140305Z","shell.execute_reply.started":"2024-03-15T12:38:28.945749Z","shell.execute_reply":"2024-03-15T12:38:44.139300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### New Category Train Test","metadata":{}},{"cell_type":"code","source":"check = NewCategoryTrainTest()\ncheck.add_condition_new_categories_less_or_equal()\n\nresult = check.run(ds_train, ds_test)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:38:44.141900Z","iopub.execute_input":"2024-03-15T12:38:44.142227Z","iopub.status.idle":"2024-03-15T12:39:02.928503Z","shell.execute_reply.started":"2024-03-15T12:38:44.142201Z","shell.execute_reply":"2024-03-15T12:39:02.926861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Label Correlation Change","metadata":{}},{"cell_type":"code","source":"check = FeatureLabelCorrelationChange(ppscore_params={'sample': 20000})\ncheck.add_condition_feature_pps_difference_less_than(0.1)\n\nresult = check.run(ds_train, ds_test)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:39:12.712283Z","iopub.execute_input":"2024-03-15T12:39:12.713049Z","iopub.status.idle":"2024-03-15T12:41:11.198205Z","shell.execute_reply.started":"2024-03-15T12:39:12.713002Z","shell.execute_reply":"2024-03-15T12:41:11.196373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Drift","metadata":{}},{"cell_type":"code","source":"check = FeatureDrift()\ncheck.add_condition_drift_score_less_than(0.2, 0.2)\n\nresult = check.run(ds_train, ds_test)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:41:53.605500Z","iopub.execute_input":"2024-03-15T12:41:53.605965Z","iopub.status.idle":"2024-03-15T12:44:08.464867Z","shell.execute_reply.started":"2024-03-15T12:41:53.605927Z","shell.execute_reply":"2024-03-15T12:44:08.463705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Evaluation","metadata":{}},{"cell_type":"code","source":"params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 8,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 500,\n    \"colsample_bytree\": 0.8, \n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n}\n\nmodel = lgb.LGBMClassifier(**params)\nmodel.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:45:15.548316Z","iopub.execute_input":"2024-03-15T12:45:15.548698Z","iopub.status.idle":"2024-03-15T12:53:05.246226Z","shell.execute_reply.started":"2024-03-15T12:45:15.548670Z","shell.execute_reply":"2024-03-15T12:53:05.245295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prediction Drift","metadata":{}},{"cell_type":"code","source":"check = PredictionDrift()\n\nresult = check.run(train_dataset=ds_train, test_dataset=ds_test, model=model)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:58:26.443967Z","iopub.execute_input":"2024-03-15T12:58:26.444441Z","iopub.status.idle":"2024-03-15T12:58:46.722031Z","shell.execute_reply.started":"2024-03-15T12:58:26.444409Z","shell.execute_reply":"2024-03-15T12:58:46.721096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Roc Report","metadata":{}},{"cell_type":"code","source":"check = RocReport()\n\nresult = check.run(ds_data, model)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:59:04.165111Z","iopub.execute_input":"2024-03-15T12:59:04.165578Z","iopub.status.idle":"2024-03-15T12:59:11.856838Z","shell.execute_reply.started":"2024-03-15T12:59:04.165522Z","shell.execute_reply":"2024-03-15T12:59:11.855130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Segment Performance","metadata":{}},{"cell_type":"code","source":"check = SegmentPerformance(feature_1='month_decision', feature_2='max_dpdmaxdateyear_596T')\n\nresult = check.run(ds_test, model)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T12:59:13.149364Z","iopub.execute_input":"2024-03-15T12:59:13.149761Z","iopub.status.idle":"2024-03-15T13:00:51.592211Z","shell.execute_reply.started":"2024-03-15T12:59:13.149732Z","shell.execute_reply":"2024-03-15T13:00:51.590549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boosting Overfit","metadata":{}},{"cell_type":"code","source":"check = BoostingOverfit()\n\nresult = check.run(ds_train, ds_test, model)\nresult.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T13:14:56.798763Z","iopub.execute_input":"2024-03-15T13:14:56.800970Z","iopub.status.idle":"2024-03-15T13:35:42.728469Z","shell.execute_reply.started":"2024-03-15T13:14:56.800901Z","shell.execute_reply":"2024-03-15T13:35:42.727148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}