{"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8325835,"sourceType":"datasetVersion","datasetId":4945134}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

  | EDA with Sweetviz

\n
","metadata":{}},{"cell_type":"markdown","source":"
\n

  | EDA with \"depth=0\" data

\n
","metadata":{}},{"cell_type":"markdown","source":"
\n

 1 | Install Libraries

\n
","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"#!pip install sweetviz","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport glob\nfrom pathlib import Path\nfrom datetime import datetime\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nimport joblib\nimport lightgbm as lgb\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:32.736633Z","start_time":"2024-05-05T17:13:32.733898Z"},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-05-05T18:37:05.709119Z","iopub.execute_input":"2024-05-05T18:37:05.709510Z","iopub.status.idle":"2024-05-05T18:37:05.716881Z","shell.execute_reply.started":"2024-05-05T18:37:05.709483Z","shell.execute_reply":"2024-05-05T18:37:05.715411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

 2 | Data Preparation

\n
","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"folder_path = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train\" \n\ntrain_dataframe = {}\n\nfor filepath in glob.glob(f\"{folder_path}/train_*.parquet\"):\n filename = filepath.split('/')[-1].split('.')[0]\n \n train_dataframe[filename] = pl.read_parquet(filepath)\n\ntrain_base_df = train_dataframe['train_base']\ntrain_static_0_0_df = train_dataframe['train_static_0_0']\ntrain_static_0_1_df = train_dataframe['train_static_0_1']\ntrain_static_cb_0_df = train_dataframe['train_static_cb_0']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:46.807967Z","start_time":"2024-05-05T17:13:33.704808Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/test\" \n\ntest_dataframe = {}\n\nfor filepath in glob.glob(f\"{folder_path}/test_*.parquet\"):\n filename = filepath.split('/')[-1].split('.')[0]\n \n test_dataframe[filename] = pl.read_parquet(filepath)\n\ntest_base_df = test_dataframe['test_base']\ntest_static_0_0_df = test_dataframe['test_static_0_0']\ntest_static_0_1_df = test_dataframe['test_static_0_1']\ntest_static_0_2_df = test_dataframe['test_static_0_2']\ntest_static_cb_0_df = test_dataframe['test_static_cb_0']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:46.835931Z","start_time":"2024-05-05T17:13:46.809257Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = ((train_base_df.join(train_static_0_0_df, on=\"case_id\", how=\"left\", suffix=\"_0_0\")\n .join(train_static_0_1_df, on=\"case_id\", how=\"left\", suffix=\"_0_1\"))\n .join(train_static_cb_0_df, on=\"case_id\", how=\"left\", suffix=\"_cb_0\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:47.681054Z","start_time":"2024-05-05T17:13:46.836795Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_static_0_0_df, on=\"case_id\", how=\"left\", suffix=\"_0_0\")\n .join(test_static_0_1_df, on=\"case_id\", how=\"left\", suffix=\"_0_1\")\n .join(test_static_0_2_df, on=\"case_id\", how=\"left\", suffix=\"_0_2\")\n .join(test_static_cb_0_df, on=\"case_id\", how=\"left\", suffix=\"_cb_0\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:47.686666Z","start_time":"2024-05-05T17:13:47.682473Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize_df(df):\n optimized_df = df.with_columns(\n [\n pl.col(col).cast(pl.Int32) if df[col].dtype in [pl.Int64, pl.Int32, pl.Int16] else\n pl.col(col).cast(pl.Float32) if df[col].dtype in [pl.Float64] else\n pl.col(col).cast(pl.Categorical) for col in df.columns\n ]\n )\n return optimized_df\n\ntrain_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:48.269163Z","start_time":"2024-05-05T17:13:47.687231Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:13:48.993114Z","start_time":"2024-05-05T17:13:48.270012Z"},"execution":{"iopub.status.busy":"2024-05-05T18:36:42.463371Z","iopub.execute_input":"2024-05-05T18:36:42.463747Z","iopub.status.idle":"2024-05-05T18:36:42.856448Z","shell.execute_reply.started":"2024-05-05T18:36:42.463721Z","shell.execute_reply":"2024-05-05T18:36:42.854951Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

 3 | EDA with sweetviz

\n
","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"import sweetviz as sv\nfeature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_depth0_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:21.824196Z","start_time":"2024-05-05T15:34:31.320020Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_depth0_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:21.829706Z","start_time":"2024-05-05T15:38:21.825913Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_depth0_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-05T18:52:21.743652Z","iopub.execute_input":"2024-05-05T18:52:21.744045Z","iopub.status.idle":"2024-05-05T18:52:22.312688Z","shell.execute_reply.started":"2024-05-05T18:52:21.744016Z","shell.execute_reply":"2024-05-05T18:52:22.310904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"applprev_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_applprev_1_0_df = train_dataframe['train_applprev_1_0']\ntrain_applprev_1_1_df = train_dataframe['train_applprev_1_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:21.832543Z","start_time":"2024-05-05T15:38:21.830726Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_applprev_1_0_df = test_dataframe['test_applprev_1_0']\ntest_applprev_1_1_df = test_dataframe['test_applprev_1_1']\ntest_applprev_1_2_df = test_dataframe['test_applprev_1_2']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:21.836822Z","start_time":"2024-05-05T15:38:21.835007Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = ((train_base_df.join(train_applprev_1_0_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\")\n .join(train_applprev_1_1_df, on=\"case_id\", how=\"left\", suffix=\"_1_1\")))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:23.932137Z","start_time":"2024-05-05T15:38:21.837450Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_applprev_1_0_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\")\n .join(test_applprev_1_1_df, on=\"case_id\", how=\"left\", suffix=\"_1_1\")\n .join(test_applprev_1_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_2\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:23.938395Z","start_time":"2024-05-05T15:38:23.933557Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:24.836325Z","start_time":"2024-05-05T15:38:23.939288Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:38:26.153366Z","start_time":"2024-05-05T15:38:24.837474Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_applprev_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.236351Z","start_time":"2024-05-05T15:38:26.154244Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_applprev_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.241707Z","start_time":"2024-05-05T15:41:12.237850Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_applprev_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-05T18:52:24.790549Z","iopub.execute_input":"2024-05-05T18:52:24.790961Z","iopub.status.idle":"2024-05-05T18:52:24.909504Z","shell.execute_reply.started":"2024-05-05T18:52:24.790929Z","shell.execute_reply":"2024-05-05T18:52:24.908433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"other_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_other_1_df = train_dataframe['train_other_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.244302Z","start_time":"2024-05-05T15:41:12.242648Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_other_1_df = test_dataframe['test_other_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.246970Z","start_time":"2024-05-05T15:41:12.245054Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_other_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.290916Z","start_time":"2024-05-05T15:41:12.247594Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_other_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.296390Z","start_time":"2024-05-05T15:41:12.292136Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.323421Z","start_time":"2024-05-05T15:41:12.298025Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:12.411940Z","start_time":"2024-05-05T15:41:12.324974Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_other_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:16.943933Z","start_time":"2024-05-05T15:41:12.413603Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_other_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:16.949243Z","start_time":"2024-05-05T15:41:16.945540Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_other_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-05T18:52:28.233480Z","iopub.execute_input":"2024-05-05T18:52:28.234073Z","iopub.status.idle":"2024-05-05T18:52:28.258746Z","shell.execute_reply.started":"2024-05-05T18:52:28.234042Z","shell.execute_reply":"2024-05-05T18:52:28.257801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"tax_registry_a_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_tax_registry_a_1_df = train_dataframe['train_tax_registry_a_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:16.952333Z","start_time":"2024-05-05T15:41:16.950283Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_tax_registry_a_1_df = test_dataframe['test_tax_registry_a_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:16.954997Z","start_time":"2024-05-05T15:41:16.953205Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_tax_registry_a_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:17.239716Z","start_time":"2024-05-05T15:41:16.955801Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_tax_registry_a_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:17.254792Z","start_time":"2024-05-05T15:41:17.249659Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:17.347769Z","start_time":"2024-05-05T15:41:17.255884Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:17.469682Z","start_time":"2024-05-05T15:41:17.349078Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_tax_registry_a_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.461575Z","start_time":"2024-05-05T15:41:17.470818Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_tax_registry_a_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.466692Z","start_time":"2024-05-05T15:41:25.463001Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_tax_registry_a_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-05T18:52:35.049709Z","iopub.execute_input":"2024-05-05T18:52:35.050125Z","iopub.status.idle":"2024-05-05T18:52:35.074449Z","shell.execute_reply.started":"2024-05-05T18:52:35.050087Z","shell.execute_reply":"2024-05-05T18:52:35.073348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"tax_registry_b_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_tax_registry_b_1_df = train_dataframe['train_tax_registry_b_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.469425Z","start_time":"2024-05-05T15:41:25.467639Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_tax_registry_b_1_df = test_dataframe['test_tax_registry_b_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.472369Z","start_time":"2024-05-05T15:41:25.470178Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_tax_registry_b_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.527604Z","start_time":"2024-05-05T15:41:25.473151Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_tax_registry_b_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.534534Z","start_time":"2024-05-05T15:41:25.528957Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.582484Z","start_time":"2024-05-05T15:41:25.536352Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:41:25.649658Z","start_time":"2024-05-05T15:41:25.583605Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_tax_registry_b_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.350255Z","start_time":"2024-05-05T15:41:25.650530Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_tax_registry_b_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.357453Z","start_time":"2024-05-05T15:49:46.351954Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_tax_registry_b_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-05T18:52:39.146145Z","iopub.execute_input":"2024-05-05T18:52:39.147435Z","iopub.status.idle":"2024-05-05T18:52:39.169696Z","shell.execute_reply.started":"2024-05-05T18:52:39.147387Z","shell.execute_reply":"2024-05-05T18:52:39.168310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"tax_registry_c_1\" data

\n
\n* Test data was missing.","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_tax_registry_c_1_df = train_dataframe['train_tax_registry_c_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.361071Z","start_time":"2024-05-05T15:49:46.358581Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_tax_registry_c_1_df = test_dataframe['test_tax_registry_c_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.364508Z","start_time":"2024-05-05T15:49:46.361960Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tax_registry_c_1_df.head()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.368614Z","start_time":"2024-05-05T15:49:46.365362Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_tax_registry_c_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.528773Z","start_time":"2024-05-05T15:49:46.369514Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_tax_registry_c_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:49:46.980459Z","start_time":"2024-05-05T15:49:46.529834Z"},"_kg_hide-output":true,"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_tax_registry_c_1_train_test.html')","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_tax_registry_c_1_train_test.html', width='100%', height='900px'))","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test data was missing","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"credit_bureau_a_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_credit_bureau_a_1_0_df = train_dataframe['train_credit_bureau_a_1_0']\ntrain_credit_bureau_a_1_1_df = train_dataframe['train_credit_bureau_a_1_1']\ntrain_credit_bureau_a_1_2_df = train_dataframe['train_credit_bureau_a_1_2']\ntrain_credit_bureau_a_1_3_df = train_dataframe['train_credit_bureau_a_1_3']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:51:10.181664Z","start_time":"2024-05-05T15:51:10.178504Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_credit_bureau_a_1_0_df = test_dataframe['test_credit_bureau_a_1_0']\ntest_credit_bureau_a_1_1_df = test_dataframe['test_credit_bureau_a_1_1']\ntest_credit_bureau_a_1_2_df = test_dataframe['test_credit_bureau_a_1_2']\ntest_credit_bureau_a_1_3_df = test_dataframe['test_credit_bureau_a_1_3']\ntest_credit_bureau_a_1_4_df = test_dataframe['test_credit_bureau_a_1_4']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:51:10.408237Z","start_time":"2024-05-05T15:51:10.406032Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_credit_bureau_a_1_0_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\")\n .join(train_credit_bureau_a_1_1_df, on=\"case_id\", how=\"left\", suffix=\"_1_1\")\n .join(train_credit_bureau_a_1_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_2\")\n .join(train_credit_bureau_a_1_3_df, on=\"case_id\", how=\"left\", suffix=\"_1_3\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:51:33.170417Z","start_time":"2024-05-05T15:51:10.554060Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_credit_bureau_a_1_0_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\")\n .join(test_credit_bureau_a_1_1_df, on=\"case_id\", how=\"left\", suffix=\"_1_1\")\n .join(test_credit_bureau_a_1_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_2\")\n .join(test_credit_bureau_a_1_3_df, on=\"case_id\", how=\"left\", suffix=\"_1_3\")\n .join(test_credit_bureau_a_1_4_df, on=\"case_id\", how=\"left\", suffix=\"_1_4\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:51:33.180963Z","start_time":"2024-05-05T15:51:33.173426Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:51:41.346917Z","start_time":"2024-05-05T15:51:33.181564Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T15:51:53.164568Z","start_time":"2024-05-05T15:51:41.349766Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_credit_bureau_a_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:36.897470Z","start_time":"2024-05-05T15:51:53.165343Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_credit_bureau_a_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:36.903366Z","start_time":"2024-05-05T16:02:36.898970Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_other_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-05T18:52:48.785913Z","iopub.execute_input":"2024-05-05T18:52:48.786314Z","iopub.status.idle":"2024-05-05T18:52:48.810734Z","shell.execute_reply.started":"2024-05-05T18:52:48.786281Z","shell.execute_reply":"2024-05-05T18:52:48.809545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"credit_bureau_b_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_credit_bureau_b_1_df = train_dataframe['train_credit_bureau_b_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:36.906744Z","start_time":"2024-05-05T16:02:36.904388Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_credit_bureau_b_1_df = test_dataframe['test_credit_bureau_b_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:36.910380Z","start_time":"2024-05-05T16:02:36.908288Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_credit_bureau_b_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:37.277284Z","start_time":"2024-05-05T16:02:36.911461Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_credit_bureau_b_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:37.283292Z","start_time":"2024-05-05T16:02:37.280512Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:37.366747Z","start_time":"2024-05-05T16:02:37.284099Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:02:37.766333Z","start_time":"2024-05-05T16:02:37.367994Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_credit_bureau_b_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.449887Z","start_time":"2024-05-05T16:02:37.767268Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_credit_bureau_b_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.455445Z","start_time":"2024-05-05T16:03:01.451096Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_tax_registry_b_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.458899Z","start_time":"2024-05-05T16:03:01.456566Z"},"execution":{"iopub.status.busy":"2024-05-05T18:52:52.941201Z","iopub.execute_input":"2024-05-05T18:52:52.941616Z","iopub.status.idle":"2024-05-05T18:52:52.963331Z","shell.execute_reply.started":"2024-05-05T18:52:52.941588Z","shell.execute_reply":"2024-05-05T18:52:52.962474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"deposit_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_deposit_1_df = train_dataframe['train_deposit_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.462341Z","start_time":"2024-05-05T16:03:01.460033Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_deposit_1_df = test_dataframe['test_deposit_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.464541Z","start_time":"2024-05-05T16:03:01.463035Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_deposit_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.525885Z","start_time":"2024-05-05T16:03:01.465280Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_deposit_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.530314Z","start_time":"2024-05-05T16:03:01.527619Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.558228Z","start_time":"2024-05-05T16:03:01.531146Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:01.610728Z","start_time":"2024-05-05T16:03:01.559149Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_deposit_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:05.872277Z","start_time":"2024-05-05T16:03:01.611736Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_deposit_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:05.878585Z","start_time":"2024-05-05T16:03:05.874842Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_deposit_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:05.881095Z","start_time":"2024-05-05T16:03:05.879533Z"},"execution":{"iopub.status.busy":"2024-05-05T18:52:56.978612Z","iopub.execute_input":"2024-05-05T18:52:56.978991Z","iopub.status.idle":"2024-05-05T18:52:57.002269Z","shell.execute_reply.started":"2024-05-05T18:52:56.978963Z","shell.execute_reply":"2024-05-05T18:52:57.000991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"person_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_person_1_df = train_dataframe['train_person_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:05.884320Z","start_time":"2024-05-05T16:03:05.882215Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_person_1_df = test_dataframe['test_person_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:05.887163Z","start_time":"2024-05-05T16:03:05.885226Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_person_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:06.137951Z","start_time":"2024-05-05T16:03:05.889337Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_person_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:06.147476Z","start_time":"2024-05-05T16:03:06.145099Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:06.312746Z","start_time":"2024-05-05T16:03:06.148489Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:06.677596Z","start_time":"2024-05-05T16:03:06.313504Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_person_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.386300Z","start_time":"2024-05-05T16:03:06.678412Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_person_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.392091Z","start_time":"2024-05-05T16:03:51.387538Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_person_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.394675Z","start_time":"2024-05-05T16:03:51.392898Z"},"execution":{"iopub.status.busy":"2024-05-05T18:53:01.052992Z","iopub.execute_input":"2024-05-05T18:53:01.053411Z","iopub.status.idle":"2024-05-05T18:53:01.094714Z","shell.execute_reply.started":"2024-05-05T18:53:01.053380Z","shell.execute_reply":"2024-05-05T18:53:01.093498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"debitcard_1\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_debitcard_1_df = train_dataframe['train_debitcard_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.396871Z","start_time":"2024-05-05T16:03:51.395331Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_debitcard_1_df = test_dataframe['test_debitcard_1']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.399171Z","start_time":"2024-05-05T16:03:51.397626Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_debitcard_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.454272Z","start_time":"2024-05-05T16:03:51.399879Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_debitcard_1_df, on=\"case_id\", how=\"left\", suffix=\"_1\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.461291Z","start_time":"2024-05-05T16:03:51.456113Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.508102Z","start_time":"2024-05-05T16:03:51.462684Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:51.549897Z","start_time":"2024-05-05T16:03:51.509335Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_debitcard_1_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:56.949038Z","start_time":"2024-05-05T16:03:51.551009Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_debitcard_1_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:56.955033Z","start_time":"2024-05-05T16:03:56.951433Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_debitcard_1_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:56.957486Z","start_time":"2024-05-05T16:03:56.955970Z"},"execution":{"iopub.status.busy":"2024-05-05T18:53:05.362845Z","iopub.execute_input":"2024-05-05T18:53:05.363235Z","iopub.status.idle":"2024-05-05T18:53:05.387977Z","shell.execute_reply.started":"2024-05-05T18:53:05.363207Z","shell.execute_reply":"2024-05-05T18:53:05.386755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"applprev_2\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_applprev_2_df = train_dataframe['train_applprev_2']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:56.959952Z","start_time":"2024-05-05T16:03:56.958368Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_applprev_2_df = test_dataframe['test_applprev_2']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:56.963093Z","start_time":"2024-05-05T16:03:56.960810Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_applprev_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:57.396602Z","start_time":"2024-05-05T16:03:56.964133Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_applprev_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:57.399767Z","start_time":"2024-05-05T16:03:57.397535Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:57.699612Z","start_time":"2024-05-05T16:03:57.400415Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:03:58.009511Z","start_time":"2024-05-05T16:03:57.700589Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_applprev_2_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.079908Z","start_time":"2024-05-05T16:03:58.010427Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_applprev_2_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.090401Z","start_time":"2024-05-05T16:04:18.083555Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_applprev_2_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.093035Z","start_time":"2024-05-05T16:04:18.091462Z"},"execution":{"iopub.status.busy":"2024-05-05T18:53:09.733803Z","iopub.execute_input":"2024-05-05T18:53:09.735568Z","iopub.status.idle":"2024-05-05T18:53:09.760749Z","shell.execute_reply.started":"2024-05-05T18:53:09.735516Z","shell.execute_reply":"2024-05-05T18:53:09.759614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"person_2\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_person_2_df = train_dataframe['train_person_2']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.097835Z","start_time":"2024-05-05T16:04:18.094057Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_person_2_df = test_dataframe['test_person_2']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.100816Z","start_time":"2024-05-05T16:04:18.098855Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_person_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.245665Z","start_time":"2024-05-05T16:04:18.101874Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_person_2_df, on=\"case_id\", how=\"left\", suffix=\"_1_0\"))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.250931Z","start_time":"2024-05-05T16:04:18.247550Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.303440Z","start_time":"2024-05-05T16:04:18.252065Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:18.398078Z","start_time":"2024-05-05T16:04:18.304750Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_person_2_train_test.html')","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:35.355453Z","start_time":"2024-05-05T16:04:18.398879Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_person_2_train_test.html', width='100%', height='900px'))","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:35.360671Z","start_time":"2024-05-05T16:04:35.356896Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\n\ntry:\n with open('/kaggle/input/home-credit-2024-sweetviz/sweetViz_person_2_train_test.html', 'r') as file:\n html_content = file.read()\n\n styled_html = f'
{html_content}
'\n display(HTML(styled_html))\nexcept Exception as e:\n print(f\"Error: {e}\")","metadata":{"ExecuteTime":{"end_time":"2024-05-05T16:04:35.365483Z","start_time":"2024-05-05T16:04:35.362068Z"},"execution":{"iopub.status.busy":"2024-05-05T18:53:13.553724Z","iopub.execute_input":"2024-05-05T18:53:13.554104Z","iopub.status.idle":"2024-05-05T18:53:13.580859Z","shell.execute_reply.started":"2024-05-05T18:53:13.554075Z","shell.execute_reply":"2024-05-05T18:53:13.579500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"credit_bureau_a_2\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_credit_bureau_a_2_0_df = train_dataframe['train_credit_bureau_a_2_0']\ntrain_credit_bureau_a_2_1_df = train_dataframe['train_credit_bureau_a_2_1']\ntrain_credit_bureau_a_2_2_df = train_dataframe['train_credit_bureau_a_2_2']\ntrain_credit_bureau_a_2_3_df = train_dataframe['train_credit_bureau_a_2_3']\ntrain_credit_bureau_a_2_4_df = train_dataframe['train_credit_bureau_a_2_4']\ntrain_credit_bureau_a_2_5_df = train_dataframe['train_credit_bureau_a_2_5']\ntrain_credit_bureau_a_2_6_df = train_dataframe['train_credit_bureau_a_2_6']\ntrain_credit_bureau_a_2_7_df = train_dataframe['train_credit_bureau_a_2_7']\ntrain_credit_bureau_a_2_8_df = train_dataframe['train_credit_bureau_a_2_8']\ntrain_credit_bureau_a_2_9_df = train_dataframe['train_credit_bureau_a_2_9']\ntrain_credit_bureau_a_2_10_df = train_dataframe['train_credit_bureau_a_2_10']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:14:19.471032Z","start_time":"2024-05-05T17:14:19.468488Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_credit_bureau_a_2_0_df = test_dataframe['test_credit_bureau_a_2_0']\ntest_credit_bureau_a_2_1_df = test_dataframe['test_credit_bureau_a_2_1']\ntest_credit_bureau_a_2_2_df = test_dataframe['test_credit_bureau_a_2_2']\ntest_credit_bureau_a_2_3_df = test_dataframe['test_credit_bureau_a_2_3']\ntest_credit_bureau_a_2_4_df = test_dataframe['test_credit_bureau_a_2_4']\ntest_credit_bureau_a_2_5_df = test_dataframe['test_credit_bureau_a_2_5']\ntest_credit_bureau_a_2_6_df = test_dataframe['test_credit_bureau_a_2_6']\ntest_credit_bureau_a_2_7_df = test_dataframe['test_credit_bureau_a_2_7']\ntest_credit_bureau_a_2_8_df = test_dataframe['test_credit_bureau_a_2_8']\ntest_credit_bureau_a_2_9_df = test_dataframe['test_credit_bureau_a_2_9']\ntest_credit_bureau_a_2_10_df = test_dataframe['test_credit_bureau_a_2_10']\ntest_credit_bureau_a_2_11_df = test_dataframe['test_credit_bureau_a_2_11']","metadata":{"ExecuteTime":{"end_time":"2024-05-05T17:14:19.767428Z","start_time":"2024-05-05T17:14:19.764895Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_credit_bureau_a_2_0_df, on=\"case_id\", how=\"left\", suffix=\"_2_0\")\n .join(train_credit_bureau_a_2_1_df, on=\"case_id\", how=\"left\", suffix=\"_2_1\")\n .join(train_credit_bureau_a_2_2_df, on=\"case_id\", how=\"left\", suffix=\"_2_2\")\n .join(train_credit_bureau_a_2_3_df, on=\"case_id\", how=\"left\", suffix=\"_2_3\")\n .join(train_credit_bureau_a_2_4_df, on=\"case_id\", how=\"left\", suffix=\"_2_4\")\n .join(train_credit_bureau_a_2_5_df, on=\"case_id\", how=\"left\", suffix=\"_2_5\")\n .join(train_credit_bureau_a_2_6_df, on=\"case_id\", how=\"left\", suffix=\"_2_6\")\n .join(train_credit_bureau_a_2_7_df, on=\"case_id\", how=\"left\", suffix=\"_2_7\")\n .join(train_credit_bureau_a_2_8_df, on=\"case_id\", how=\"left\", suffix=\"_2_8\")\n .join(train_credit_bureau_a_2_9_df, on=\"case_id\", how=\"left\", suffix=\"_2_9\")\n .join(train_credit_bureau_a_2_10_df, on=\"case_id\", how=\"left\", suffix=\"_2_10\"))","metadata":{"jupyter":{"is_executing":true},"ExecuteTime":{"start_time":"2024-05-05T17:14:20.041853Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_credit_bureau_a_2_0_df, on=\"case_id\", how=\"left\", suffix=\"_2_0\")\n .join(test_credit_bureau_a_2_1_df, on=\"case_id\", how=\"left\", suffix=\"_2_1\")\n .join(test_credit_bureau_a_2_2_df, on=\"case_id\", how=\"left\", suffix=\"_2_2\")\n .join(test_credit_bureau_a_2_3_df, on=\"case_id\", how=\"left\", suffix=\"_2_3\")\n .join(test_credit_bureau_a_2_4_df, on=\"case_id\", how=\"left\", suffix=\"_2_4\")\n .join(test_credit_bureau_a_2_5_df, on=\"case_id\", how=\"left\", suffix=\"_2_5\")\n .join(test_credit_bureau_a_2_6_df, on=\"case_id\", how=\"left\", suffix=\"_2_6\")\n .join(test_credit_bureau_a_2_7_df, on=\"case_id\", how=\"left\", suffix=\"_2_7\")\n .join(test_credit_bureau_a_2_8_df, on=\"case_id\", how=\"left\", suffix=\"_2_8\")\n .join(test_credit_bureau_a_2_9_df, on=\"case_id\", how=\"left\", suffix=\"_2_9\")\n .join(test_credit_bureau_a_2_10_df, on=\"case_id\", how=\"left\", suffix=\"_2_10\")\n .join(test_credit_bureau_a_2_11_df, on=\"case_id\", how=\"left\", suffix=\"_2_11\"))","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_credit_bureau_a_2_train_test.html')","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_credit_bureau_a_2_train_test.html', width='100%', height='900px'))","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO","metadata":{"jupyter":{"is_executing":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

  | EDA with \"credit_bureau_b_2\" data

\n
","metadata":{}},{"cell_type":"code","source":"train_base_df = train_dataframe['train_base']\ntrain_credit_bureau_b_2_df = train_dataframe['train_credit_bureau_b_2']","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_base_df = test_dataframe['test_base']\ntest_credit_bureau_b_2_df = test_dataframe['test_credit_bureau_b_2']","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = (train_base_df.join(train_credit_bureau_b_2_df, on=\"case_id\", how=\"left\", suffix=\"_2\"))","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final_df = (test_base_df\n .join(test_credit_bureau_b_2_df, on=\"case_id\", how=\"left\", suffix=\"_2\"))","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df = optimize_df(train_final_df)\ntest_final_df = optimize_df(test_final_df)","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_df_pandas = train_final_df.to_pandas()\ntest_final_df_pandas = test_final_df.to_pandas()","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_config = sv.FeatureConfig(skip=['case_id']) \nmy_report = sv.compare([train_final_df_pandas, \"Train Data\"], [test_final_df_pandas, \"Test Data\"], \"target\", feature_config, pairwise_analysis='off')\nmy_report.show_html('sweetViz_credit_bureau_b_2_train_test.html')","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import IFrame\n\ndisplay(IFrame(src='sweetViz_credit_bureau_b_2_train_test.html', width='100%', height='900px'))","metadata":{"jupyter":{"is_executing":true},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO","metadata":{},"execution_count":null,"outputs":[]}]}