{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-23T04:49:26.914312Z","iopub.execute_input":"2024-02-23T04:49:26.915009Z","iopub.status.idle":"2024-02-23T04:49:28.505089Z","shell.execute_reply.started":"2024-02-23T04:49:26.914949Z","shell.execute_reply":"2024-02-23T04:49:28.503867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-02-23T04:59:47.445103Z","iopub.execute_input":"2024-02-23T04:59:47.446177Z","iopub.status.idle":"2024-02-23T04:59:47.452873Z","shell.execute_reply.started":"2024-02-23T04:59:47.446134Z","shell.execute_reply":"2024-02-23T04:59:47.451365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T04:59:47.629875Z","iopub.execute_input":"2024-02-23T04:59:47.630313Z","iopub.status.idle":"2024-02-23T04:59:47.637193Z","shell.execute_reply.started":"2024-02-23T04:59:47.630279Z","shell.execute_reply":"2024-02-23T04:59:47.634806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dic = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')\ndata_dic.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T04:59:47.840383Z","iopub.execute_input":"2024-02-23T04:59:47.841064Z","iopub.status.idle":"2024-02-23T04:59:47.896429Z","shell.execute_reply.started":"2024-02-23T04:59:47.841019Z","shell.execute_reply":"2024-02-23T04:59:47.894888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv')\nbase.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:00:06.119849Z","iopub.execute_input":"2024-02-23T05:00:06.120366Z","iopub.status.idle":"2024-02-23T05:00:07.570234Z","shell.execute_reply.started":"2024-02-23T05:00:06.120330Z","shell.execute_reply":"2024-02-23T05:00:07.569123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base.describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:00:07.824615Z","iopub.execute_input":"2024-02-23T05:00:07.825039Z","iopub.status.idle":"2024-02-23T05:00:08.008510Z","shell.execute_reply.started":"2024-02-23T05:00:07.825007Z","shell.execute_reply":"2024-02-23T05:00:08.007254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:20:22.294623Z","iopub.execute_input":"2024-02-22T10:20:22.294929Z","iopub.status.idle":"2024-02-22T10:20:22.318397Z","shell.execute_reply.started":"2024-02-22T10:20:22.294903Z","shell.execute_reply":"2024-02-22T10:20:22.316492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0 = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_0.csv')\nstat_0.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:00:14.114308Z","iopub.execute_input":"2024-02-23T05:00:14.115287Z","iopub.status.idle":"2024-02-23T05:00:49.363635Z","shell.execute_reply.started":"2024-02-23T05:00:14.115247Z","shell.execute_reply":"2024-02-23T05:00:49.362082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['lastst_736L'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:31:15.594133Z","iopub.execute_input":"2024-02-23T05:31:15.594974Z","iopub.status.idle":"2024-02-23T05:31:15.829758Z","shell.execute_reply.started":"2024-02-23T05:31:15.594909Z","shell.execute_reply":"2024-02-23T05:31:15.827813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['lastst_736L'].dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:31:19.319756Z","iopub.execute_input":"2024-02-23T05:31:19.320163Z","iopub.status.idle":"2024-02-23T05:31:19.329734Z","shell.execute_reply.started":"2024-02-23T05:31:19.320131Z","shell.execute_reply":"2024-02-23T05:31:19.328063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['lastst_736L'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:31:34.203496Z","iopub.execute_input":"2024-02-23T05:31:34.204552Z","iopub.status.idle":"2024-02-23T05:31:34.264972Z","shell.execute_reply.started":"2024-02-23T05:31:34.204446Z","shell.execute_reply":"2024-02-23T05:31:34.263651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['lastst_736L'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:31:48.468414Z","iopub.execute_input":"2024-02-23T05:31:48.469210Z","iopub.status.idle":"2024-02-23T05:31:48.610313Z","shell.execute_reply.started":"2024-02-23T05:31:48.469161Z","shell.execute_reply":"2024-02-23T05:31:48.609416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['maininc_215A'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:34:07.717975Z","iopub.execute_input":"2024-02-23T05:34:07.718373Z","iopub.status.idle":"2024-02-23T05:34:07.795176Z","shell.execute_reply.started":"2024-02-23T05:34:07.718344Z","shell.execute_reply":"2024-02-23T05:34:07.793379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(stat_0['maininc_215A'])","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:34:20.086755Z","iopub.execute_input":"2024-02-23T05:34:20.087177Z","iopub.status.idle":"2024-02-23T05:34:21.473751Z","shell.execute_reply.started":"2024-02-23T05:34:20.087144Z","shell.execute_reply":"2024-02-23T05:34:21.471816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(stat_0['interestrate_311L'], stat_0['credamount_770A'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:13:48.383102Z","iopub.execute_input":"2024-02-23T05:13:48.383869Z","iopub.status.idle":"2024-02-23T05:13:50.590299Z","shell.execute_reply.started":"2024-02-23T05:13:48.383829Z","shell.execute_reply":"2024-02-23T05:13:50.588317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(stat_0[stat_0['interestrate_311L'].notnull()]['interestrate_311L'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:11:50.002522Z","iopub.execute_input":"2024-02-23T05:11:50.002943Z","iopub.status.idle":"2024-02-23T05:11:51.548654Z","shell.execute_reply.started":"2024-02-23T05:11:50.002904Z","shell.execute_reply":"2024-02-23T05:11:51.546432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0[stat_0['eir_270L'] == 0].head()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:01:14.832871Z","iopub.execute_input":"2024-02-23T05:01:14.833289Z","iopub.status.idle":"2024-02-23T05:01:15.373756Z","shell.execute_reply.started":"2024-02-23T05:01:14.833259Z","shell.execute_reply":"2024-02-23T05:01:15.372690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dic[data_dic['Variable'].isin(stat_0.columns)].to_csv('variable_desc.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:42:37.141550Z","iopub.execute_input":"2024-02-22T10:42:37.141881Z","iopub.status.idle":"2024-02-22T10:42:37.150307Z","shell.execute_reply.started":"2024-02-22T10:42:37.141855Z","shell.execute_reply":"2024-02-22T10:42:37.149524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dic[data_dic['Variable'].isin(stat_0.columns)]","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:20:36.532710Z","iopub.execute_input":"2024-02-22T10:20:36.532994Z","iopub.status.idle":"2024-02-22T10:20:36.555907Z","shell.execute_reply.started":"2024-02-22T10:20:36.532969Z","shell.execute_reply":"2024-02-22T10:20:36.554345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0.count()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:20:36.557204Z","iopub.execute_input":"2024-02-22T10:20:36.557542Z","iopub.status.idle":"2024-02-22T10:20:37.776019Z","shell.execute_reply.started":"2024-02-22T10:20:36.557515Z","shell.execute_reply":"2024-02-22T10:20:37.775182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:20:37.777230Z","iopub.execute_input":"2024-02-22T10:20:37.777578Z","iopub.status.idle":"2024-02-22T10:20:37.790562Z","shell.execute_reply.started":"2024-02-22T10:20:37.777549Z","shell.execute_reply":"2024-02-22T10:20:37.789063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_drop = []\nrow_count = len(stat_0.index)\ndrop_threshold = 0.1 * row_count\n\nfor c in stat_0.columns:\n    if stat_0[c].count() < drop_threshold:\n        print(stat_0[c].count() , '<>', drop_threshold)\n        to_drop.append(c)\nprint(to_drop)\nstat_0.drop(columns=to_drop, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:35:08.319879Z","iopub.execute_input":"2024-02-22T10:35:08.320265Z","iopub.status.idle":"2024-02-22T10:35:10.160755Z","shell.execute_reply.started":"2024-02-22T10:35:08.320234Z","shell.execute_reply":"2024-02-22T10:35:10.159097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0.count()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:35:44.477923Z","iopub.execute_input":"2024-02-22T10:35:44.478266Z","iopub.status.idle":"2024-02-22T10:35:45.535589Z","shell.execute_reply.started":"2024-02-22T10:35:44.478239Z","shell.execute_reply":"2024-02-22T10:35:45.534183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.matshow(stat_0.select_dtypes(include='number').corr())","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:50:47.079442Z","iopub.execute_input":"2024-02-22T10:50:47.079792Z","iopub.status.idle":"2024-02-22T10:51:08.559845Z","shell.execute_reply.started":"2024-02-22T10:50:47.079766Z","shell.execute_reply":"2024-02-22T10:51:08.558491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:51:08.562005Z","iopub.execute_input":"2024-02-22T10:51:08.562405Z","iopub.status.idle":"2024-02-22T10:51:08.569004Z","shell.execute_reply.started":"2024-02-22T10:51:08.562375Z","shell.execute_reply":"2024-02-22T10:51:08.566537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import missingno as msno","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:51:08.570244Z","iopub.execute_input":"2024-02-22T10:51:08.570626Z","iopub.status.idle":"2024-02-22T10:51:08.581536Z","shell.execute_reply.started":"2024-02-22T10:51:08.570594Z","shell.execute_reply":"2024-02-22T10:51:08.580204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.matrix(stat_0)","metadata":{"execution":{"iopub.status.busy":"2024-02-22T10:51:08.583851Z","iopub.execute_input":"2024-02-22T10:51:08.584175Z","iopub.status.idle":"2024-02-22T10:51:36.527171Z","shell.execute_reply.started":"2024-02-22T10:51:08.584146Z","shell.execute_reply":"2024-02-22T10:51:36.526187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['maxdbddpdtollast6m_4187119P'].count()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:50:03.604853Z","iopub.execute_input":"2024-02-23T05:50:03.605414Z","iopub.status.idle":"2024-02-23T05:50:03.616269Z","shell.execute_reply.started":"2024-02-23T05:50:03.605340Z","shell.execute_reply":"2024-02-23T05:50:03.614935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number where both 6m and 12m are both null\nlen(stat_0[(stat_0['maxdbddpdtollast6m_4187119P'].isnull()) & (stat_0['maxdbddpdtollast12m_3658940P'].isnull())].index)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:52:32.589219Z","iopub.execute_input":"2024-02-23T05:52:32.589650Z","iopub.status.idle":"2024-02-23T05:52:33.404294Z","shell.execute_reply.started":"2024-02-23T05:52:32.589619Z","shell.execute_reply":"2024-02-23T05:52:33.402787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last6m_null_12m_not_null = stat_0[(stat_0['maxdbddpdtollast6m_4187119P'].isnull()) & (stat_0['maxdbddpdtollast12m_3658940P'].notnull())]\nplt.hist(last6m_null_12m_not_null['maxdbddpdtollast12m_3658940P'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:53:51.861124Z","iopub.execute_input":"2024-02-23T05:53:51.861628Z","iopub.status.idle":"2024-02-23T05:53:52.436477Z","shell.execute_reply.started":"2024-02-23T05:53:51.861590Z","shell.execute_reply":"2024-02-23T05:53:52.435210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_0['price_1097A'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T06:25:47.767995Z","iopub.execute_input":"2024-02-23T06:25:47.768397Z","iopub.status.idle":"2024-02-23T06:25:47.887240Z","shell.execute_reply.started":"2024-02-23T06:25:47.768368Z","shell.execute_reply":"2024-02-23T06:25:47.885988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}