{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv\nimport re\nimport seaborn as sns\nimport gc,os\nimport warnings\nfrom IPython.display import HTML\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-05-09T07:29:06.982023Z","iopub.execute_input":"2024-05-09T07:29:06.982657Z","iopub.status.idle":"2024-05-09T07:29:09.917389Z","shell.execute_reply.started":"2024-05-09T07:29:06.982620Z","shell.execute_reply":"2024-05-09T07:29:09.916282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install sweetviz","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T07:29:09.919623Z","iopub.execute_input":"2024-05-09T07:29:09.920275Z","iopub.status.idle":"2024-05-09T07:29:26.865536Z","shell.execute_reply.started":"2024-05-09T07:29:09.920230Z","shell.execute_reply":"2024-05-09T07:29:26.864330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Use sweetviz auto EDA library : https://www.geeksforgeeks.org/sweetviz-automated-exploratory-data-analysis-eda/ ,https://towardsdatascience.com/powerful-eda-exploratory-data-analysis-in-just-two-lines-of-code-using-sweetviz-6c943d32f34","metadata":{}},{"cell_type":"markdown","source":"# All files listed below are found in both .csv and .parquet formats.\n\n# static : EMI Payment related details\n# applprev : Credit payment previous data.\n# other : debit payment purchage data.\n# tax_registry : Tax deductions amount tracked by the government registry .\n# credit_bureau : track record of credit & payment details.\n# deposit : deposits of user.\n# person : personal family, house , employee status\n# debitcard : debit card transctions details","metadata":{}},{"cell_type":"markdown","source":"#
Base Tables analysis 🧘🏿
","metadata":{}},{"cell_type":"code","source":"train_base = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv')\nprint(\"start date in training data : \",train_base[\"date_decision\"].min(),\"end date in training data : \",train_base[\"date_decision\"].max(),end='\\n\\n')\nprint(train_base.shape)\nprint(\"Sample of Training Base Table below : \")\ntrain_base.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T07:29:32.582396Z","iopub.execute_input":"2024-05-09T07:29:32.582836Z","iopub.status.idle":"2024-05-09T07:29:34.713280Z","shell.execute_reply.started":"2024-05-09T07:29:32.582799Z","shell.execute_reply":"2024-05-09T07:29:34.712407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sweetviz as sv\nreport = sv.analyze([train_base,\"train_base\"],target_feat='target')\n\nreport.show_html(filepath='Report.html')\n\ntry:\n with open(os.path.join(os.getcwd(),\"Report.html\"), 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:44:18.959638Z","iopub.execute_input":"2024-05-09T07:44:18.960407Z","iopub.status.idle":"2024-05-09T07:44:31.206860Z","shell.execute_reply.started":"2024-05-09T07:44:18.960360Z","shell.execute_reply":"2024-05-09T07:44:31.205277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_base.csv')\nprint(test_base.shape)\nprint(\"start date in testing data : \",test_base[\"date_decision\"].min(),\"end date in testing data : \",test_base[\"date_decision\"].max(),end='\\n\\n')\nprint(\"Sample of Testing Base Table below : \")\ntest_base.head(5)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T07:50:50.958814Z","iopub.execute_input":"2024-05-09T07:50:50.959262Z","iopub.status.idle":"2024-05-09T07:50:50.981092Z","shell.execute_reply.started":"2024-05-09T07:50:50.959230Z","shell.execute_reply":"2024-05-09T07:50:50.979773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature definations file\nfd = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')\nprint(fd.shape)\nfd = dict(zip(fd.Variable,fd.Description))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-09T07:50:49.084447Z","iopub.execute_input":"2024-05-09T07:50:49.085302Z","iopub.status.idle":"2024-05-09T07:50:49.103566Z","shell.execute_reply.started":"2024-05-09T07:50:49.085259Z","shell.execute_reply":"2024-05-09T07:50:49.102300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#
Very highly Unbalanced Dataset 🧘🏿
","metadata":{}},{"cell_type":"code","source":"vc = train_base[\"target\"].value_counts()\nvc = pd.DataFrame({\"target\":vc.index,\"target_counts\":vc.values,\"as_percentage\":((vc/train_base.shape[0])*100).values})\nprint(vc)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:48:54.559177Z","iopub.execute_input":"2024-05-09T07:48:54.559643Z","iopub.status.idle":"2024-05-09T07:48:54.584800Z","shell.execute_reply.started":"2024-05-09T07:48:54.559604Z","shell.execute_reply":"2024-05-09T07:48:54.583698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(vc,x='target',y='as_percentage',)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:48:55.503696Z","iopub.execute_input":"2024-05-09T07:48:55.504321Z","iopub.status.idle":"2024-05-09T07:48:55.744500Z","shell.execute_reply.started":"2024-05-09T07:48:55.504290Z","shell.execute_reply":"2024-05-09T07:48:55.742887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *****\n\n#
static file Analysis
\n\n\n# static_0 - 2 training data , 3 testing data (ex : train_static_0_0.csv)\n Properties: depth=0, internal data source ","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:49:12.539563Z","iopub.execute_input":"2024-05-09T07:49:12.540106Z","iopub.status.idle":"2024-05-09T07:49:13.071615Z","shell.execute_reply.started":"2024-05-09T07:49:12.540069Z","shell.execute_reply":"2024-05-09T07:49:13.070010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data\nts = pd.DataFrame()\nfor i in range(2):\n ts = pd.concat([ts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_{i}.parquet')],ignore_index=True)\nprint(ts.shape)\nts.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:49:13.123921Z","iopub.execute_input":"2024-05-09T07:49:13.124335Z","iopub.status.idle":"2024-05-09T07:49:21.436275Z","shell.execute_reply.started":"2024-05-09T07:49:13.124304Z","shell.execute_reply":"2024-05-09T07:49:21.435113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/1526659\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:50:58.554363Z","iopub.execute_input":"2024-05-09T07:50:58.555340Z","iopub.status.idle":"2024-05-09T07:51:03.629204Z","shell.execute_reply.started":"2024-05-09T07:50:58.555287Z","shell.execute_reply":"2024-05-09T07:51:03.627918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nhigh_corr = []\nprint(\"Column pairs with correlation greater than 0.8 or less than -0.8:\")\nfor pair in high_correlation_pairs:\n v1 = pair[0].split('_')[0]\n v2 = pair[1].split(\"_\")[0]\n if v1 != v2 and re.split('[0-9]',v1)[0] != re.split('[0-9]', v2)[0] and v1[:3] != v2[:3]:\n high_corr.append(pair)\nlen(high_corr)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:51:13.824318Z","iopub.execute_input":"2024-05-09T07:51:13.824753Z","iopub.status.idle":"2024-05-09T07:52:13.860821Z","shell.execute_reply.started":"2024-05-09T07:51:13.824720Z","shell.execute_reply":"2024-05-09T07:52:13.859432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data\ntts = pd.DataFrame()\nfor i in range(3):\n tts = pd.concat([tts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_0_{str(i)}.parquet')],ignore_index=True)\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:54:01.928198Z","iopub.execute_input":"2024-05-09T07:54:01.929412Z","iopub.status.idle":"2024-05-09T07:54:02.106737Z","shell.execute_reply.started":"2024-05-09T07:54:01.929348Z","shell.execute_reply":"2024-05-09T07:54:02.105383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data case id which are not in base case id's\nstr(set(tts[\"case_id\"].unique()) - set(test_base['case_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:54:21.239390Z","iopub.execute_input":"2024-05-09T07:54:21.239889Z","iopub.status.idle":"2024-05-09T07:54:21.249702Z","shell.execute_reply.started":"2024-05-09T07:54:21.239853Z","shell.execute_reply":"2024-05-09T07:54:21.248133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data case id which are not in base data case id's\nstr(set(ts[\"case_id\"].unique()) - set(train_base['case_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:50:37.828655Z","iopub.execute_input":"2024-05-09T07:50:37.829370Z","iopub.status.idle":"2024-05-09T07:50:38.866828Z","shell.execute_reply.started":"2024-05-09T07:50:37.829325Z","shell.execute_reply":"2024-05-09T07:50:38.865741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no duplicate case id's\nts[\"case_id\"].duplicated().sum(),len(ts[\"case_id\"].unique()),ts.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-09T07:50:38.868696Z","iopub.execute_input":"2024-05-09T07:50:38.869061Z","iopub.status.idle":"2024-05-09T07:50:38.947116Z","shell.execute_reply.started":"2024-05-09T07:50:38.869008Z","shell.execute_reply":"2024-05-09T07:50:38.945960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for static file : \n# 1. Data is about : emi payment realted deatils.\n# 2. 21 columns with more than 70 % null values.(suggestion : fill with zero or empty string)\n# 3. 74 columns with more than 80 % correlated columns. (suggestion : Don't remove columns)\n\n###########################################\n\n# static_cb_0 - 1 training ,1 testing data (ex: train_static_cb_0.csv)\n Properties: depth=0, external data source ","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:51:47.581853Z","iopub.execute_input":"2024-03-29T10:51:47.583240Z","iopub.status.idle":"2024-03-29T10:51:48.014525Z","shell.execute_reply.started":"2024-03-29T10:51:47.583189Z","shell.execute_reply":"2024-03-29T10:51:48.012709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_cb_0.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:51:48.019430Z","iopub.execute_input":"2024-03-29T10:51:48.020017Z","iopub.status.idle":"2024-03-29T10:51:51.020571Z","shell.execute_reply.started":"2024-03-29T10:51:48.019970Z","shell.execute_reply":"2024-03-29T10:51:51.018573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:11.299831Z","iopub.execute_input":"2024-03-29T10:52:11.300712Z","iopub.status.idle":"2024-03-29T10:52:13.818847Z","shell.execute_reply.started":"2024-03-29T10:52:11.300667Z","shell.execute_reply":"2024-03-29T10:52:13.817580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\nsns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()])","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:32.746432Z","iopub.execute_input":"2024-03-29T10:52:32.746978Z","iopub.status.idle":"2024-03-29T10:52:36.587139Z","shell.execute_reply.started":"2024-03-29T10:52:32.746941Z","shell.execute_reply":"2024-03-29T10:52:36.585311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_static_cb_0.parquet')\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:40.801645Z","iopub.execute_input":"2024-03-29T10:52:40.802243Z","iopub.status.idle":"2024-03-29T10:52:40.862464Z","shell.execute_reply.started":"2024-03-29T10:52:40.802202Z","shell.execute_reply":"2024-03-29T10:52:40.860587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data case id which are not in base case id's\nstr(set(tts[\"case_id\"].unique()) - set(test_base['case_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:41.789840Z","iopub.execute_input":"2024-03-29T10:52:41.790542Z","iopub.status.idle":"2024-03-29T10:52:41.800307Z","shell.execute_reply.started":"2024-03-29T10:52:41.790495Z","shell.execute_reply":"2024-03-29T10:52:41.799251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data case id which are not in base data case id's\nlen(set(train_base['case_id'].unique()) - set(tts[\"case_id\"].unique())),train_base.shape[0] - tts.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:42.320951Z","iopub.execute_input":"2024-03-29T10:52:42.321453Z","iopub.status.idle":"2024-03-29T10:52:43.023144Z","shell.execute_reply.started":"2024-03-29T10:52:42.321417Z","shell.execute_reply":"2024-03-29T10:52:43.021935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no duplicate case id's\ntts[\"case_id\"].duplicated().sum(),len(tts[\"case_id\"].unique()),tts.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:43.025730Z","iopub.execute_input":"2024-03-29T10:52:43.026168Z","iopub.status.idle":"2024-03-29T10:52:43.046892Z","shell.execute_reply.started":"2024-03-29T10:52:43.026124Z","shell.execute_reply":"2024-03-29T10:52:43.040492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for static cb file : \n# 1. Data is about : emi payment realted deatils.\n# 2. 30 columns with more than 70 % null values.(suggestion : fill with zero or empty string)\n# 3. 13 columns with more than 80 % correlated columns. (suggestion : Don't remove columns)\n# 4. their are 26183 training data case id's not in base training data case id's \n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# *************\n\n#
applprev Analysis
\n\n\n# applprev_1 - 2 trianing files, 3 testing files\nProperties: depth=1, internal data source","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:45.011496Z","iopub.execute_input":"2024-03-29T10:52:45.013395Z","iopub.status.idle":"2024-03-29T10:52:45.597049Z","shell.execute_reply.started":"2024-03-29T10:52:45.013274Z","shell.execute_reply":"2024-03-29T10:52:45.590374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training data\nts = pd.DataFrame()\nfor i in range(2):\n ts = pd.concat([ts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_1_{str(i)}.parquet')],ignore_index=True)\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:52:47.040127Z","iopub.execute_input":"2024-03-29T10:52:47.041004Z","iopub.status.idle":"2024-03-29T10:53:03.550540Z","shell.execute_reply.started":"2024-03-29T10:52:47.040961Z","shell.execute_reply":"2024-03-29T10:53:03.548178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:03.560013Z","iopub.execute_input":"2024-03-29T10:53:03.560585Z","iopub.status.idle":"2024-03-29T10:53:16.251910Z","shell.execute_reply.started":"2024-03-29T10:53:03.560543Z","shell.execute_reply":"2024-03-29T10:53:16.250429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\nsns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()])","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:16.253255Z","iopub.execute_input":"2024-03-29T10:53:16.253969Z","iopub.status.idle":"2024-03-29T10:53:25.145114Z","shell.execute_reply.started":"2024-03-29T10:53:16.253934Z","shell.execute_reply":"2024-03-29T10:53:25.142934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.DataFrame()\nfor i in range(3):\n tts = pd.concat([tts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_applprev_1_{str(i)}.parquet')],ignore_index=True)\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:25.149821Z","iopub.execute_input":"2024-03-29T10:53:25.150270Z","iopub.status.idle":"2024-03-29T10:53:25.241621Z","shell.execute_reply.started":"2024-03-29T10:53:25.150240Z","shell.execute_reply":"2024-03-29T10:53:25.238944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data case id which are not in base case id's\nprint(tts.shape[0]-test_base.shape[0])\nstr(set(tts[\"case_id\"].unique()) - set(test_base['case_id'].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:25.243453Z","iopub.execute_input":"2024-03-29T10:53:25.244727Z","iopub.status.idle":"2024-03-29T10:53:25.256058Z","shell.execute_reply.started":"2024-03-29T10:53:25.244692Z","shell.execute_reply":"2024-03-29T10:53:25.254798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data case id which are not in base data case id's\nlen(set(train_base['case_id'].unique()) - set(ts[\"case_id\"].unique())),ts.shape[0] - train_base.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:25.258084Z","iopub.execute_input":"2024-03-29T10:53:25.259630Z","iopub.status.idle":"2024-03-29T10:53:26.331193Z","shell.execute_reply.started":"2024-03-29T10:53:25.259596Z","shell.execute_reply":"2024-03-29T10:53:26.328475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# many duplicate case id's????\nts[\"case_id\"].duplicated().sum(),len(ts[\"case_id\"].unique()),ts.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:26.333243Z","iopub.execute_input":"2024-03-29T10:53:26.333826Z","iopub.status.idle":"2024-03-29T10:53:26.537502Z","shell.execute_reply.started":"2024-03-29T10:53:26.333790Z","shell.execute_reply":"2024-03-29T10:53:26.535893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# over all duplicated data\nts.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:53:26.539712Z","iopub.execute_input":"2024-03-29T10:53:26.542318Z","iopub.status.idle":"2024-03-29T10:54:09.596421Z","shell.execute_reply.started":"2024-03-29T10:53:26.542246Z","shell.execute_reply":"2024-03-29T10:54:09.594777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# applprev_2","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:09.598826Z","iopub.execute_input":"2024-03-29T10:54:09.599318Z","iopub.status.idle":"2024-03-29T10:54:09.997672Z","shell.execute_reply.started":"2024-03-29T10:54:09.599283Z","shell.execute_reply":"2024-03-29T10:54:09.996742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training data\nts = pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_2.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:10.002988Z","iopub.execute_input":"2024-03-29T10:54:10.003749Z","iopub.status.idle":"2024-03-29T10:54:14.952945Z","shell.execute_reply.started":"2024-03-29T10:54:10.003699Z","shell.execute_reply":"2024-03-29T10:54:14.951167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:55.492902Z","iopub.execute_input":"2024-03-29T10:54:55.493392Z","iopub.status.idle":"2024-03-29T10:54:58.919026Z","shell.execute_reply.started":"2024-03-29T10:54:55.493355Z","shell.execute_reply":"2024-03-29T10:54:58.917247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for applprev file : \n# 1. Data is about : credit payment previous data deatils ? \n# 2. 9 columns with more than 70 % null values.(suggestion : fill with zero or empty string)\n# 3. 2 columns with more than 80 % correlated columns. (suggestion : Don't remove columns)\n# 4. their are 305137 training data case id's not in base training data case id's ?\n# 5. since its previous credit card trasction data so we have many duplicate case id's\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# *********\n#
other Analysis
\n\n# other_1 - 1 training file and 1 testing file\nProperties: depth=1, internal data source","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.534318Z","iopub.status.idle":"2024-03-29T10:54:18.534809Z","shell.execute_reply.started":"2024-03-29T10:54:18.534575Z","shell.execute_reply":"2024-03-29T10:54:18.534594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_other_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.537739Z","iopub.status.idle":"2024-03-29T10:54:18.539178Z","shell.execute_reply.started":"2024-03-29T10:54:18.538869Z","shell.execute_reply":"2024-03-29T10:54:18.538898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ts.columns:\n try:\n print(col,\"defination : \",fd[col])\n except:\n print(\"column not in definations : \", col)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.540775Z","iopub.status.idle":"2024-03-29T10:54:18.541389Z","shell.execute_reply.started":"2024-03-29T10:54:18.541052Z","shell.execute_reply":"2024-03-29T10:54:18.541076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nts_null","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.543435Z","iopub.status.idle":"2024-03-29T10:54:18.544638Z","shell.execute_reply.started":"2024-03-29T10:54:18.544259Z","shell.execute_reply":"2024-03-29T10:54:18.544286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\nsns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()].rename(fd))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.547633Z","iopub.status.idle":"2024-03-29T10:54:18.548236Z","shell.execute_reply.started":"2024-03-29T10:54:18.547928Z","shell.execute_reply":"2024-03-29T10:54:18.547955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_other_1.parquet')\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.551608Z","iopub.status.idle":"2024-03-29T10:54:18.553163Z","shell.execute_reply.started":"2024-03-29T10:54:18.552803Z","shell.execute_reply":"2024-03-29T10:54:18.552839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data case id which are not in base case id's\nprint(tts.shape[0]-test_base.shape[0])\nstr(set(test_base['case_id'].unique()) - set(tts[\"case_id\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.557064Z","iopub.status.idle":"2024-03-29T10:54:18.557634Z","shell.execute_reply.started":"2024-03-29T10:54:18.557386Z","shell.execute_reply":"2024-03-29T10:54:18.557410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data case id which are not in base data case id's\nlen(set(train_base['case_id'].unique()) - set(ts[\"case_id\"].unique())),ts.shape[0] - train_base.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.560146Z","iopub.status.idle":"2024-03-29T10:54:18.560696Z","shell.execute_reply.started":"2024-03-29T10:54:18.560456Z","shell.execute_reply":"2024-03-29T10:54:18.560479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for other file : \n# 1. Data is about : debit payment data deatils ? \n# 2. 2 columns with more than 80 % correlated columns. (suggestion : Don't remove columns)\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# ********** ************\n#
tax_registry Analysis
\n\n# train_tax_registry_{}_1 - Properties: depth=1, external data source, Tax registry provider A,B,C","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.562927Z","iopub.status.idle":"2024-03-29T10:54:18.563446Z","shell.execute_reply.started":"2024-03-29T10:54:18.563184Z","shell.execute_reply":"2024-03-29T10:54:18.563203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_a_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.565385Z","iopub.status.idle":"2024-03-29T10:54:18.565829Z","shell.execute_reply.started":"2024-03-29T10:54:18.565630Z","shell.execute_reply":"2024-03-29T10:54:18.565647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ts.columns:\n try:\n print(col,\"defination : \",fd[col])\n except:\n print(\"column not in definations : \", col)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.567215Z","iopub.status.idle":"2024-03-29T10:54:18.567691Z","shell.execute_reply.started":"2024-03-29T10:54:18.567485Z","shell.execute_reply":"2024-03-29T10:54:18.567504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/51109\nts_null","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.569797Z","iopub.status.idle":"2024-03-29T10:54:18.570752Z","shell.execute_reply.started":"2024-03-29T10:54:18.570042Z","shell.execute_reply":"2024-03-29T10:54:18.570059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\n#sns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()])","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.573310Z","iopub.status.idle":"2024-03-29T10:54:18.573841Z","shell.execute_reply.started":"2024-03-29T10:54:18.573613Z","shell.execute_reply":"2024-03-29T10:54:18.573634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_tax_registry_a_1.parquet')\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.578815Z","iopub.status.idle":"2024-03-29T10:54:18.579431Z","shell.execute_reply.started":"2024-03-29T10:54:18.579149Z","shell.execute_reply":"2024-03-29T10:54:18.579171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data case id which are not in base data case id's\nlen(set(train_base['case_id'].unique()) - set(ts[\"case_id\"].unique())),ts.shape[0] - train_base.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.582423Z","iopub.status.idle":"2024-03-29T10:54:18.582941Z","shell.execute_reply.started":"2024-03-29T10:54:18.582720Z","shell.execute_reply":"2024-03-29T10:54:18.582739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tax_registry_b Analysis","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.584983Z","iopub.status.idle":"2024-03-29T10:54:18.585514Z","shell.execute_reply.started":"2024-03-29T10:54:18.585253Z","shell.execute_reply":"2024-03-29T10:54:18.585271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_b_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.587717Z","iopub.status.idle":"2024-03-29T10:54:18.588179Z","shell.execute_reply.started":"2024-03-29T10:54:18.587969Z","shell.execute_reply":"2024-03-29T10:54:18.587986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.590420Z","iopub.status.idle":"2024-03-29T10:54:18.590947Z","shell.execute_reply.started":"2024-03-29T10:54:18.590654Z","shell.execute_reply":"2024-03-29T10:54:18.590671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ts.columns:\n try:\n print(col,\"defination : \",fd[col])\n except:\n print(\"column not in definations : \", col)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.596597Z","iopub.status.idle":"2024-03-29T10:54:18.597422Z","shell.execute_reply.started":"2024-03-29T10:54:18.597053Z","shell.execute_reply":"2024-03-29T10:54:18.597082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_tax_registry_b_1.parquet')\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.599774Z","iopub.status.idle":"2024-03-29T10:54:18.600319Z","shell.execute_reply.started":"2024-03-29T10:54:18.600080Z","shell.execute_reply":"2024-03-29T10:54:18.600100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# tax_registry_c Analysis","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.603653Z","iopub.status.idle":"2024-03-29T10:54:18.604153Z","shell.execute_reply.started":"2024-03-29T10:54:18.603941Z","shell.execute_reply":"2024-03-29T10:54:18.603961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_c_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.605775Z","iopub.status.idle":"2024-03-29T10:54:18.606218Z","shell.execute_reply.started":"2024-03-29T10:54:18.606002Z","shell.execute_reply":"2024-03-29T10:54:18.606019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ts.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.607187Z","iopub.status.idle":"2024-03-29T10:54:18.607656Z","shell.execute_reply.started":"2024-03-29T10:54:18.607444Z","shell.execute_reply":"2024-03-29T10:54:18.607463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ts.columns:\n try:\n print(col,\"defination : \",fd[col])\n except:\n print(\"column not in definations : \", col)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.610837Z","iopub.status.idle":"2024-03-29T10:54:18.611444Z","shell.execute_reply.started":"2024-03-29T10:54:18.611167Z","shell.execute_reply":"2024-03-29T10:54:18.611188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_tax_registry_c_1.parquet')\nprint(tts.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.612582Z","iopub.status.idle":"2024-03-29T10:54:18.613016Z","shell.execute_reply.started":"2024-03-29T10:54:18.612812Z","shell.execute_reply":"2024-03-29T10:54:18.612830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for tax registry file : \n# 1. Data is about : 3 columns - Tax deductions amount tracked by the government registry , date, name, num_group1\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# *****************\n#
credit_bureau Analysis
\n\n\n# credit_bureau_{a,b}_{1,2}_{} - training data , depth 1,2 \n\n# A credit bureau, is an organization that collects and researches individual credit information and sells it to creditors for a fee","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:15:59.476088Z","iopub.execute_input":"2024-03-30T04:15:59.476482Z","iopub.status.idle":"2024-03-30T04:15:59.568415Z","shell.execute_reply.started":"2024-03-30T04:15:59.476454Z","shell.execute_reply":"2024-03-30T04:15:59.567249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training data\nts = pd.DataFrame()\nfor i in range(2):\n ts = pd.concat([ts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_1_{str(i)}.parquet')],ignore_index=True)\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:13:41.157345Z","iopub.execute_input":"2024-03-30T04:13:41.157739Z","iopub.status.idle":"2024-03-30T04:14:03.365375Z","shell.execute_reply.started":"2024-03-30T04:13:41.157709Z","shell.execute_reply":"2024-03-30T04:14:03.364347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:16:02.743411Z","iopub.execute_input":"2024-03-30T04:16:02.743785Z","iopub.status.idle":"2024-03-30T04:16:13.636189Z","shell.execute_reply.started":"2024-03-30T04:16:02.743755Z","shell.execute_reply":"2024-03-30T04:16:13.635143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\nsns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()].rename(fd))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:16:13.638219Z","iopub.execute_input":"2024-03-30T04:16:13.638988Z","iopub.status.idle":"2024-03-30T04:17:03.593781Z","shell.execute_reply.started":"2024-03-30T04:16:13.638952Z","shell.execute_reply":"2024-03-30T04:17:03.592543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.DataFrame()\nfor i in range(5):\n tts = pd.concat([tts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_a_1_{str(i)}.parquet')],ignore_index=True)\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.625552Z","iopub.status.idle":"2024-03-29T10:54:18.626007Z","shell.execute_reply.started":"2024-03-29T10:54:18.625803Z","shell.execute_reply":"2024-03-29T10:54:18.625820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data case id which are not in base data case id's\nlen(set(train_base['case_id'].unique()) - set(ts[\"case_id\"].unique())),ts.shape[0] - train_base.shape[0]","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.628809Z","iopub.status.idle":"2024-03-29T10:54:18.629355Z","shell.execute_reply.started":"2024-03-29T10:54:18.629092Z","shell.execute_reply":"2024-03-29T10:54:18.629110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data case id which are not in base case id's\nprint(tts.shape[0]-test_base.shape[0])\nstr(set(test_base['case_id'].unique()) - set(tts[\"case_id\"].unique()))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.631886Z","iopub.status.idle":"2024-03-29T10:54:18.632879Z","shell.execute_reply.started":"2024-03-29T10:54:18.632542Z","shell.execute_reply":"2024-03-29T10:54:18.632583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# credit_bureau_b Analysis","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.635447Z","iopub.status.idle":"2024-03-29T10:54:18.635957Z","shell.execute_reply.started":"2024-03-29T10:54:18.635733Z","shell.execute_reply":"2024-03-29T10:54:18.635753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.639202Z","iopub.status.idle":"2024-03-29T10:54:18.639742Z","shell.execute_reply.started":"2024-03-29T10:54:18.639505Z","shell.execute_reply":"2024-03-29T10:54:18.639526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.643431Z","iopub.status.idle":"2024-03-29T10:54:18.643984Z","shell.execute_reply.started":"2024-03-29T10:54:18.643745Z","shell.execute_reply":"2024-03-29T10:54:18.643765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\nsns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()].rename(fd))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.648646Z","iopub.status.idle":"2024-03-29T10:54:18.649435Z","shell.execute_reply.started":"2024-03-29T10:54:18.649053Z","shell.execute_reply":"2024-03-29T10:54:18.649082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_b_1.parquet')\nprint(tts.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.651122Z","iopub.status.idle":"2024-03-29T10:54:18.651766Z","shell.execute_reply.started":"2024-03-29T10:54:18.651466Z","shell.execute_reply":"2024-03-29T10:54:18.651493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# credit_bureau_a_2 \n\na vertion 2","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.654293Z","iopub.status.idle":"2024-03-29T10:54:18.654953Z","shell.execute_reply.started":"2024-03-29T10:54:18.654678Z","shell.execute_reply":"2024-03-29T10:54:18.654700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training data\nts = pd.DataFrame()\nfor i in range(2):\n ts = pd.concat([ts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_2_{str(i)}.parquet')],ignore_index=True)\nprint(ts.shape)\nts.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.656653Z","iopub.status.idle":"2024-03-29T10:54:18.657178Z","shell.execute_reply.started":"2024-03-29T10:54:18.656951Z","shell.execute_reply":"2024-03-29T10:54:18.656971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.661862Z","iopub.status.idle":"2024-03-29T10:54:18.662623Z","shell.execute_reply.started":"2024-03-29T10:54:18.662234Z","shell.execute_reply":"2024-03-29T10:54:18.662256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corealtion columns\ncorrelation_matrix = ts.select_dtypes(exclude='object').corr()\nhigh_correlation_pairs = []\nfor i in range(len(correlation_matrix.columns)):\n for j in range(i + 1, len(correlation_matrix.columns)):\n if abs(correlation_matrix.iloc[i, j]) > 0.8:\n high_correlation_pairs.append((correlation_matrix.columns[i], correlation_matrix.columns[j]))\nprint(len(high_correlation_pairs))\nsns.heatmap(correlation_matrix.loc[dict(high_correlation_pairs).keys(),dict(high_correlation_pairs).values()].rename(fd))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.665655Z","iopub.status.idle":"2024-03-29T10:54:18.666189Z","shell.execute_reply.started":"2024-03-29T10:54:18.665965Z","shell.execute_reply":"2024-03-29T10:54:18.665983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.DataFrame()\nfor i in range(11):\n tts = pd.concat([tts,pd.read_parquet(f'/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_a_2_{str(i)}.parquet')],ignore_index=True)\nprint(tts.shape)\ntts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.670730Z","iopub.status.idle":"2024-03-29T10:54:18.671324Z","shell.execute_reply.started":"2024-03-29T10:54:18.671078Z","shell.execute_reply":"2024-03-29T10:54:18.671100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# credit_bureau_b_2","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.673302Z","iopub.status.idle":"2024-03-29T10:54:18.673871Z","shell.execute_reply.started":"2024-03-29T10:54:18.673640Z","shell.execute_reply":"2024-03-29T10:54:18.673661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_2.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.677961Z","iopub.status.idle":"2024-03-29T10:54:18.678806Z","shell.execute_reply.started":"2024-03-29T10:54:18.678321Z","shell.execute_reply":"2024-03-29T10:54:18.678492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing data\ntts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test/test_credit_bureau_b_2.parquet')\nprint(tts.shape)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.684947Z","iopub.status.idle":"2024-03-29T10:54:18.685554Z","shell.execute_reply.started":"2024-03-29T10:54:18.685286Z","shell.execute_reply":"2024-03-29T10:54:18.685307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for credit bureau file : \n# 1. Data is about : various credit bureau tracking user previous credit payments.\n# 2. we have 3 different credit bureau user data .\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# *********\n#
deposit_1
\nProperties: depth=1, internal data source","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:22:28.170276Z","iopub.execute_input":"2024-03-30T04:22:28.170750Z","iopub.status.idle":"2024-03-30T04:22:28.273156Z","shell.execute_reply.started":"2024-03-30T04:22:28.170718Z","shell.execute_reply":"2024-03-30T04:22:28.271959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_deposit_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:22:28.873492Z","iopub.execute_input":"2024-03-30T04:22:28.873882Z","iopub.status.idle":"2024-03-30T04:22:30.079649Z","shell.execute_reply.started":"2024-03-30T04:22:28.873849Z","shell.execute_reply":"2024-03-30T04:22:30.078647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ts.columns:\n try:\n print(col,\"defination : \",fd[col])\n except:\n print(\"column not in definations : \", col)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T04:22:32.873564Z","iopub.execute_input":"2024-03-30T04:22:32.873966Z","iopub.status.idle":"2024-03-30T04:22:32.879935Z","shell.execute_reply.started":"2024-03-30T04:22:32.873936Z","shell.execute_reply":"2024-03-30T04:22:32.878787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for deposit file : \n# 1. Data is about : deposits data.\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# ******\n#
person_1
\nProperties: depth=1, internal data source","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T06:28:45.289395Z","iopub.execute_input":"2024-03-30T06:28:45.289804Z","iopub.status.idle":"2024-03-30T06:28:45.377558Z","shell.execute_reply.started":"2024-03-30T06:28:45.289775Z","shell.execute_reply":"2024-03-30T06:28:45.375992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T06:28:48.166176Z","iopub.execute_input":"2024-03-30T06:28:48.167008Z","iopub.status.idle":"2024-03-30T06:28:52.149977Z","shell.execute_reply.started":"2024-03-30T06:28:48.166973Z","shell.execute_reply":"2024-03-30T06:28:52.149119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nts_null = ts.isnull().sum()/ts.shape[0]\nprint(ts_null[ts_null>=0.7].shape,ts.shape)\nts_null[ts_null>=0.7].sort_values().rename(fd).plot(kind=\"barh\",color='red',figsize=(12,6))","metadata":{"execution":{"iopub.status.busy":"2024-03-30T06:29:22.316827Z","iopub.execute_input":"2024-03-30T06:29:22.317267Z","iopub.status.idle":"2024-03-30T06:29:28.318237Z","shell.execute_reply.started":"2024-03-30T06:29:22.317232Z","shell.execute_reply":"2024-03-30T06:29:28.316982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.703451Z","iopub.status.idle":"2024-03-29T10:54:18.703937Z","shell.execute_reply.started":"2024-03-29T10:54:18.703720Z","shell.execute_reply":"2024-03-29T10:54:18.703739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_2.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T10:54:18.705576Z","iopub.status.idle":"2024-03-29T10:54:18.706034Z","shell.execute_reply.started":"2024-03-29T10:54:18.705823Z","shell.execute_reply":"2024-03-29T10:54:18.705841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for person file : \n# 1. Data is about : personal family , empoyee , house , relations details.\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# *****************\n\n#
Debitcard
","metadata":{}},{"cell_type":"code","source":"collected = gc.collect()\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainning Data\nts = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_debitcard_1.parquet')\nprint(ts.shape)\nts.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T06:44:20.959489Z","iopub.execute_input":"2024-03-30T06:44:20.960220Z","iopub.status.idle":"2024-03-30T06:44:21.374236Z","shell.execute_reply.started":"2024-03-30T06:44:20.960180Z","shell.execute_reply":"2024-03-30T06:44:21.373025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ts.columns:\n try:\n print(\"columns name : \",fd[col])\n except:\n print(\"column not having defination : \",col)","metadata":{"execution":{"iopub.status.busy":"2024-03-30T06:44:21.635689Z","iopub.execute_input":"2024-03-30T06:44:21.636136Z","iopub.status.idle":"2024-03-30T06:44:21.643131Z","shell.execute_reply.started":"2024-03-30T06:44:21.636103Z","shell.execute_reply":"2024-03-30T06:44:21.641810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"############################################\n\n# summary of EDA for debit file : \n# 1. Data is about : debit card transactions details.\n\n###########################################\n\n# *************","metadata":{}},{"cell_type":"markdown","source":"# Summary of data Understanding \n\n# Below are the challages in data:\n\n# 1. except static files all datasets don't not have all the case id's both in tran and test data.\n# 2. we can't load all data sets in one go!!\n# 3. lot of missing & correlated values.\n\n# comment your thoughts on feature selection & modelling..","metadata":{}}]}