{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"After read: https://www.kaggle.com/code/codename007/home-credit-complete-eda-feature-importance I start:\n\n- <a href='#1'>1. Introduction</a>  \n- <a href='#2'>2. Retrieving the Data</a>\n- <a href='#3'>3. Glimpse of Data</a>\n    - <a href='#3-1'>3.1 train_base.csv</a>\n    - <a href='#3-2'>3.2 train_static_0_0.csv</a>\n    - <a href='#3-3'>3.3 train_static_cb_0.csv</a>\n    - <a href='#3-4'>3.4 train_applprev_1_0.csv</a>\n    - <a href='#3-5'>3.5 train_credit_bureau_a_1_0.csv</a>\n    - <a href='#3-6'>3.6 train_credit_bureau_b_1.csv</a>\n    - <a href='#3-7'>3.7 train_debitcard_1.csv</a>\n    - <a href='#3-8'>3.8 train_deposit_1.csv</a>\n    - <a href='#3-9'>3.9 train_other_1.csv</a>\n    - <a href='#3-10'>3.10 train_person_1.csv</a>\n    - <a href='#3-11'>3.11 train_tax_registry_a_1.csv</a>\n    - <a href='#3-12'>3.12 train_tax_registry_b_1.csv</a>\n    - <a href='#3-13'>3.13 train_tax_registry_c_1.csv</a>\n    - <a href='#3-14'>3.14 train_applprev_2.csv</a>\n    - <a href='#3-15'>3.15 train_credit_bureau_a_2_0.csv</a>\n    - <a href='#3-16'>3.16 train_credit_bureau_b_2.csv</a>\n    - <a href='#3-17'>3.17 train_person_2.csv</a>\n- <a href='#4'> 4. Check for missing data</a>\n    - <a href='#4-1'>4.1 train_base.csv</a>\n    - <a href='#4-2'>4.2 train_static_0_0.csv</a>\n    - <a href='#4-3'>4.3 train_static_cb_0.csv</a>\n    - <a href='#4-4'>4.4 train_applprev_1_0.csv</a>\n    - <a href='#4-5'>4.5 train_credit_bureau_a_1_0.csv</a>\n    - <a href='#4-6'>4.6 train_credit_bureau_b_1.csv</a>\n    - <a href='#4-7'>4.7 train_debitcard_1.csv</a>\n    - <a href='#4-8'>4.8 train_deposit_1.csv</a>\n    - <a href='#4-9'>4.9 train_other_1.csv</a>\n    - <a href='#4-10'>4.10 train_person_1.csv</a>\n    - <a href='#4-11'>4.11 train_tax_registry_a_1.csv</a>\n    - <a href='#4-12'>4.12 train_tax_registry_b_1.csv</a>\n    - <a href='#4-13'>4.13 train_tax_registry_c_1.csv</a>\n    - <a href='#4-14'>4.14 train_applprev_2.csv</a>\n    - <a href='#4-15'>4.15 train_credit_bureau_a_2_0.csv</a>\n    - <a href='#4-16'>4.16 train_credit_bureau_b_2.csv</a>\n    - <a href='#4-17'>4.17 train_person_2.csv</a>\n- <a href='#5'>5. Data Exploration</a>\n    - <a href='#5-1'>5.1 train_base.csv</a>\n    - <a href='#5-2'>5.2 train_static_0_0.csv</a>\n    - <a href='#5-3'>5.3 train_static_cb_0.csv</a>\n    - <a href='#5-4'>5.4 train_applprev_1_0.csv</a>\n    - <a href='#5-5'>5.5 train_credit_bureau_a_1_0.csv</a>\n    - <a href='#5-6'>5.6 train_credit_bureau_b_1.csv</a>\n    - <a href='#5-7'>5.7 train_debitcard_1.csv</a>\n    - <a href='#5-8'>5.8 train_deposit_1.csv</a>\n    - <a href='#5-9'>5.9 train_other_1.csv</a>\n    - <a href='#5-10'>5.10 train_person_1.csv</a>\n    - <a href='#5-11'>5.11 train_tax_registry_a_1.csv</a>\n    - <a href='#5-12'>5.12 train_tax_registry_b_1.csv</a>\n    - <a href='#5-13'>5.13 train_tax_registry_c_1.csv</a>\n    - <a href='#5-14'>5.14 train_applprev_2.csv</a>\n    - <a href='#5-15'>5.15 train_credit_bureau_a_2_0.csv</a>\n    - <a href='#5-16'>5.16 train_credit_bureau_b_2.csv</a>\n    - <a href='#5-17'>5.17 train_person_2.csv</a>","metadata":{"_kg_hide-input":true}},{"cell_type":"markdown","source":"# <a id='1'>1. Introduction</a>","metadata":{}},{"cell_type":"markdown","source":"The goal of this competition is to predict which clients are more likely to default on their loans. The evaluation will favor solutions that are stable over time.\n\nYour participation may offer consumer finance providers a more reliable and longer-lasting way to assess a potential client’s default risk.","metadata":{}},{"cell_type":"markdown","source":" # <a id='2'>2. Retrieving the Data</a>","metadata":{}},{"cell_type":"code","source":"import pandas as pd # package for high-performance, easy-to-use data structures and data analysis\nimport numpy as np # fundamental package for scientific computing with Python\nimport re\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt # for plotting\nimport seaborn as sns # for making plots with seaborn\ncolor = sns.color_palette()\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.offline as offline\noffline.init_notebook_mode()\nimport cufflinks as cf\ncf.go_offline()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:05:36.890662Z","iopub.execute_input":"2024-02-22T00:05:36.891517Z","iopub.status.idle":"2024-02-22T00:05:39.794857Z","shell.execute_reply.started":"2024-02-22T00:05:36.891466Z","shell.execute_reply":"2024-02-22T00:05:39.793825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = '/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train'\nfiles = os.listdir(base_path)\nfiles.sort()\nprint(files)\nprint(f\"Total {len(files)} files\")","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:05:39.796681Z","iopub.execute_input":"2024-02-22T00:05:39.797230Z","iopub.status.idle":"2024-02-22T00:05:39.807176Z","shell.execute_reply.started":"2024-02-22T00:05:39.797199Z","shell.execute_reply":"2024-02-22T00:05:39.805954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level_0_files = []\nlevel_1_files = []\nlevel_2_files = []\npattern = r'\\d+'\nfor f in files:\n    try:\n        r = re.findall(pattern, f)\n        if re.findall(pattern, f)[0] == '1':\n            level_1_files.append(f)\n        elif re.findall(pattern, f)[0] == '2':\n            level_2_files.append(f)\n        else:\n            level_0_files.append(f)\n    except:\n        level_0_files.append(f)\nprint(f\"level 0 with [{len(level_0_files)}] files :\\n\")\nprint(level_0_files)\nprint(f\"\\nlevel 1 with [{len(level_1_files)}] files:\\n\")\nprint(level_1_files)\nprint(f\"\\nlevel 2 with [{len(level_2_files)}] files:\\n\")\nprint(level_2_files)","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:05:40.983637Z","iopub.execute_input":"2024-02-22T00:05:40.984028Z","iopub.status.idle":"2024-02-22T00:05:40.992625Z","shell.execute_reply.started":"2024-02-22T00:05:40.983998Z","shell.execute_reply":"2024-02-22T00:05:40.991693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:05:42.056445Z","iopub.execute_input":"2024-02-22T00:05:42.059108Z","iopub.status.idle":"2024-02-22T00:05:42.072770Z","shell.execute_reply.started":"2024-02-22T00:05:42.059065Z","shell.execute_reply":"2024-02-22T00:05:42.071460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # <a id='3'>3. Glimpse of Data</a>","metadata":{}},{"cell_type":"code","source":"file_dict = {\n    \n}\nfor f in files:\n    file_dict[f] = pd.read_parquet(f\"{base_path}/{f}\")\n    file_dict[f] = reduce_mem_usage(file_dict[f])","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:05:47.034001Z","iopub.execute_input":"2024-02-22T00:05:47.034454Z","iopub.status.idle":"2024-02-22T00:11:31.540270Z","shell.execute_reply.started":"2024-02-22T00:05:47.034421Z","shell.execute_reply":"2024-02-22T00:11:31.539230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-1'>3.1 train_base.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nprint(f\"{level_0_files[i]}\\n\")\n\nprint()\nprint(f\"{level_0_files[i]} shape: {file_dict[level_0_files[i]].shape} \\n\")\nprint(file_dict[level_0_files[i]].head())\nprint()\nprint(file_dict[level_0_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:16:30.340417Z","iopub.execute_input":"2024-02-17T19:16:30.340914Z","iopub.status.idle":"2024-02-17T19:16:30.350962Z","shell.execute_reply.started":"2024-02-17T19:16:30.340879Z","shell.execute_reply":"2024-02-17T19:16:30.350024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-2'>3.2 train_static_0_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 1\nprint()\nprint(f\"{level_0_files[i]} shape: {file_dict[level_0_files[i]].shape} \\n\")\nprint(file_dict[level_0_files[i]].head())\nprint()\nprint(file_dict[level_0_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:16:47.355171Z","iopub.execute_input":"2024-02-17T19:16:47.355574Z","iopub.status.idle":"2024-02-17T19:16:47.380518Z","shell.execute_reply.started":"2024-02-17T19:16:47.355544Z","shell.execute_reply":"2024-02-17T19:16:47.379191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 2\nprint()\nprint(f\"{level_0_files[i]} shape: {file_dict[level_0_files[i]].shape} \\n\")\nprint(file_dict[level_0_files[i]].head())\nprint()\nprint(file_dict[level_0_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:18:40.803183Z","iopub.execute_input":"2024-02-17T19:18:40.803590Z","iopub.status.idle":"2024-02-17T19:18:40.832856Z","shell.execute_reply.started":"2024-02-17T19:18:40.803559Z","shell.execute_reply":"2024-02-17T19:18:40.831660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-3'>3.3 train_static_cb_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 3\nprint()\nprint(f\"{level_0_files[i]} shape: {file_dict[level_0_files[i]].shape} \\n\")\nprint(file_dict[level_0_files[i]].head())\nprint()\nprint(file_dict[level_0_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:18:30.498586Z","iopub.execute_input":"2024-02-17T19:18:30.499141Z","iopub.status.idle":"2024-02-17T19:18:30.532962Z","shell.execute_reply.started":"2024-02-17T19:18:30.499096Z","shell.execute_reply":"2024-02-17T19:18:30.531663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-4'>3.4 train_applprev_1_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:18:20.353310Z","iopub.execute_input":"2024-02-17T19:18:20.353686Z","iopub.status.idle":"2024-02-17T19:18:20.378771Z","shell.execute_reply.started":"2024-02-17T19:18:20.353660Z","shell.execute_reply":"2024-02-17T19:18:20.377606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:18:08.676649Z","iopub.execute_input":"2024-02-17T19:18:08.677105Z","iopub.status.idle":"2024-02-17T19:18:08.699820Z","shell.execute_reply.started":"2024-02-17T19:18:08.677072Z","shell.execute_reply":"2024-02-17T19:18:08.698625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-5'>3.5 train_credit_bureau_a_1_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 2\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:17:57.367786Z","iopub.execute_input":"2024-02-17T19:17:57.368185Z","iopub.status.idle":"2024-02-17T19:17:57.387729Z","shell.execute_reply.started":"2024-02-17T19:17:57.368156Z","shell.execute_reply":"2024-02-17T19:17:57.386881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 3\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:03.328163Z","iopub.execute_input":"2024-02-17T19:19:03.328560Z","iopub.status.idle":"2024-02-17T19:19:03.354585Z","shell.execute_reply.started":"2024-02-17T19:19:03.328530Z","shell.execute_reply":"2024-02-17T19:19:03.352559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 4\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:10.625660Z","iopub.execute_input":"2024-02-17T19:19:10.626083Z","iopub.status.idle":"2024-02-17T19:19:10.651221Z","shell.execute_reply.started":"2024-02-17T19:19:10.626051Z","shell.execute_reply":"2024-02-17T19:19:10.649785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 5\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:18.602012Z","iopub.execute_input":"2024-02-17T19:19:18.602577Z","iopub.status.idle":"2024-02-17T19:19:18.628124Z","shell.execute_reply.started":"2024-02-17T19:19:18.602532Z","shell.execute_reply":"2024-02-17T19:19:18.626997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-6'>3.6 train_credit_bureau_b_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 6\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:32.318268Z","iopub.execute_input":"2024-02-17T19:19:32.318663Z","iopub.status.idle":"2024-02-17T19:19:32.338191Z","shell.execute_reply.started":"2024-02-17T19:19:32.318633Z","shell.execute_reply":"2024-02-17T19:19:32.337025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-7'>3.7 train_debitcard_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 7\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:41.216177Z","iopub.execute_input":"2024-02-17T19:19:41.216598Z","iopub.status.idle":"2024-02-17T19:19:41.227002Z","shell.execute_reply.started":"2024-02-17T19:19:41.216567Z","shell.execute_reply":"2024-02-17T19:19:41.225818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-8'>3.8 train_deposit_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 8\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:47.216027Z","iopub.execute_input":"2024-02-17T19:19:47.216531Z","iopub.status.idle":"2024-02-17T19:19:47.227654Z","shell.execute_reply.started":"2024-02-17T19:19:47.216491Z","shell.execute_reply":"2024-02-17T19:19:47.226326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-9'>3.9 train_other_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 9\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:19:54.053354Z","iopub.execute_input":"2024-02-17T19:19:54.053870Z","iopub.status.idle":"2024-02-17T19:19:54.070229Z","shell.execute_reply.started":"2024-02-17T19:19:54.053828Z","shell.execute_reply":"2024-02-17T19:19:54.068868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-10'>3.10 train_person_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 10\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:00.714754Z","iopub.execute_input":"2024-02-17T19:20:00.717715Z","iopub.status.idle":"2024-02-17T19:20:00.780236Z","shell.execute_reply.started":"2024-02-17T19:20:00.717571Z","shell.execute_reply":"2024-02-17T19:20:00.776712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-11'>3.11 train_tax_registry_a_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 11\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:09.036091Z","iopub.execute_input":"2024-02-17T19:20:09.037390Z","iopub.status.idle":"2024-02-17T19:20:09.062031Z","shell.execute_reply.started":"2024-02-17T19:20:09.037303Z","shell.execute_reply":"2024-02-17T19:20:09.058166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-12'>3.12 train_tax_registry_b_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 12\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:16.320274Z","iopub.execute_input":"2024-02-17T19:20:16.320682Z","iopub.status.idle":"2024-02-17T19:20:16.330952Z","shell.execute_reply.started":"2024-02-17T19:20:16.320652Z","shell.execute_reply":"2024-02-17T19:20:16.329511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-13'>3.13 train_tax_registry_c_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 13\nprint()\nprint(f\"{level_1_files[i]} shape: {file_dict[level_1_files[i]].shape} \\n\")\nprint(file_dict[level_1_files[i]].head())\nprint()\nprint(file_dict[level_1_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:22.010328Z","iopub.execute_input":"2024-02-17T19:20:22.010714Z","iopub.status.idle":"2024-02-17T19:20:22.020855Z","shell.execute_reply.started":"2024-02-17T19:20:22.010686Z","shell.execute_reply":"2024-02-17T19:20:22.019407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-14'>3.14 train_applprev_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:28.706108Z","iopub.execute_input":"2024-02-17T19:20:28.706529Z","iopub.status.idle":"2024-02-17T19:20:28.718637Z","shell.execute_reply.started":"2024-02-17T19:20:28.706498Z","shell.execute_reply":"2024-02-17T19:20:28.717314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-15'>3.15 train_credit_bureau_a_2_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 1\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:34.518426Z","iopub.execute_input":"2024-02-17T19:20:34.518806Z","iopub.status.idle":"2024-02-17T19:20:34.534607Z","shell.execute_reply.started":"2024-02-17T19:20:34.518778Z","shell.execute_reply":"2024-02-17T19:20:34.533637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 2\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:40.644010Z","iopub.execute_input":"2024-02-17T19:20:40.644543Z","iopub.status.idle":"2024-02-17T19:20:40.666381Z","shell.execute_reply.started":"2024-02-17T19:20:40.644496Z","shell.execute_reply":"2024-02-17T19:20:40.665016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 3\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:46.182904Z","iopub.execute_input":"2024-02-17T19:20:46.184024Z","iopub.status.idle":"2024-02-17T19:20:46.200062Z","shell.execute_reply.started":"2024-02-17T19:20:46.183988Z","shell.execute_reply":"2024-02-17T19:20:46.197845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 4\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:20:53.290143Z","iopub.execute_input":"2024-02-17T19:20:53.290587Z","iopub.status.idle":"2024-02-17T19:20:53.307448Z","shell.execute_reply.started":"2024-02-17T19:20:53.290554Z","shell.execute_reply":"2024-02-17T19:20:53.306042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 5\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:21:00.105717Z","iopub.execute_input":"2024-02-17T19:21:00.106146Z","iopub.status.idle":"2024-02-17T19:21:00.122483Z","shell.execute_reply.started":"2024-02-17T19:21:00.106114Z","shell.execute_reply":"2024-02-17T19:21:00.121239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 6\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:21:19.238438Z","iopub.execute_input":"2024-02-17T19:21:19.238821Z","iopub.status.idle":"2024-02-17T19:21:19.258242Z","shell.execute_reply.started":"2024-02-17T19:21:19.238793Z","shell.execute_reply":"2024-02-17T19:21:19.256839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 7\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:21:29.462692Z","iopub.execute_input":"2024-02-17T19:21:29.463115Z","iopub.status.idle":"2024-02-17T19:21:29.481402Z","shell.execute_reply.started":"2024-02-17T19:21:29.463085Z","shell.execute_reply":"2024-02-17T19:21:29.480500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 8\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:21:37.024881Z","iopub.execute_input":"2024-02-17T19:21:37.025302Z","iopub.status.idle":"2024-02-17T19:21:37.041520Z","shell.execute_reply.started":"2024-02-17T19:21:37.025273Z","shell.execute_reply":"2024-02-17T19:21:37.040249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 9\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:21:49.448197Z","iopub.execute_input":"2024-02-17T19:21:49.448699Z","iopub.status.idle":"2024-02-17T19:21:49.468800Z","shell.execute_reply.started":"2024-02-17T19:21:49.448655Z","shell.execute_reply":"2024-02-17T19:21:49.467792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 10\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:21:55.776234Z","iopub.execute_input":"2024-02-17T19:21:55.776648Z","iopub.status.idle":"2024-02-17T19:21:55.796099Z","shell.execute_reply.started":"2024-02-17T19:21:55.776615Z","shell.execute_reply":"2024-02-17T19:21:55.794857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 11\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:03.151366Z","iopub.execute_input":"2024-02-17T19:22:03.151820Z","iopub.status.idle":"2024-02-17T19:22:03.168983Z","shell.execute_reply.started":"2024-02-17T19:22:03.151777Z","shell.execute_reply":"2024-02-17T19:22:03.167923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-16'>3.16 train_credit_bureau_b_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 12\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:09.456270Z","iopub.execute_input":"2024-02-17T19:22:09.456798Z","iopub.status.idle":"2024-02-17T19:22:09.470631Z","shell.execute_reply.started":"2024-02-17T19:22:09.456758Z","shell.execute_reply":"2024-02-17T19:22:09.469440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='3-17'>3.17 train_person_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 13\nprint()\nprint(f\"{level_2_files[i]} shape: {file_dict[level_2_files[i]].shape} \\n\")\nprint(file_dict[level_2_files[i]].head())\nprint()\nprint(file_dict[level_2_files[i]].columns.values)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:15.973900Z","iopub.execute_input":"2024-02-17T19:22:15.974339Z","iopub.status.idle":"2024-02-17T19:22:15.986229Z","shell.execute_reply.started":"2024-02-17T19:22:15.974308Z","shell.execute_reply":"2024-02-17T19:22:15.985048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # <a id='4'>4. Check for missing data</a>","metadata":{}},{"cell_type":"markdown","source":"### <a id='4-1'>4.1 train_base.csv</a>","metadata":{}},{"cell_type":"code","source":"i=0\n\nprint(level_0_files[i])\nprint()\ntotal = file_dict[level_0_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_0_files[i]].isnull().sum()/file_dict[level_0_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:24.342447Z","iopub.execute_input":"2024-02-17T19:22:24.342856Z","iopub.status.idle":"2024-02-17T19:22:24.392509Z","shell.execute_reply.started":"2024-02-17T19:22:24.342825Z","shell.execute_reply":"2024-02-17T19:22:24.391200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-2'>4.2 train_static_0_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i=1\nprint(level_0_files[i])\nprint()\ntotal = file_dict[level_0_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_0_files[i]].isnull().sum()/file_dict[level_0_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:32.893772Z","iopub.execute_input":"2024-02-17T19:22:32.894157Z","iopub.status.idle":"2024-02-17T19:22:35.695507Z","shell.execute_reply.started":"2024-02-17T19:22:32.894131Z","shell.execute_reply":"2024-02-17T19:22:35.694303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=2\n\nprint(level_0_files[i])\nprint()\ntotal = file_dict[level_0_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_0_files[i]].isnull().sum()/file_dict[level_0_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:42.022280Z","iopub.execute_input":"2024-02-17T19:22:42.022816Z","iopub.status.idle":"2024-02-17T19:22:43.146781Z","shell.execute_reply.started":"2024-02-17T19:22:42.022773Z","shell.execute_reply":"2024-02-17T19:22:43.145715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-3'>4.3 train_static_cb_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i=3\n\nprint(level_0_files[i])\nprint()\ntotal = file_dict[level_0_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_0_files[i]].isnull().sum()/file_dict[level_0_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:47.799197Z","iopub.execute_input":"2024-02-17T19:22:47.799599Z","iopub.status.idle":"2024-02-17T19:22:48.766617Z","shell.execute_reply.started":"2024-02-17T19:22:47.799570Z","shell.execute_reply":"2024-02-17T19:22:48.765531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-4'>4.4 train_applprev_1_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:22:54.883614Z","iopub.execute_input":"2024-02-17T19:22:54.884025Z","iopub.status.idle":"2024-02-17T19:22:56.329040Z","shell.execute_reply.started":"2024-02-17T19:22:54.883994Z","shell.execute_reply":"2024-02-17T19:22:56.327804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:01.351339Z","iopub.execute_input":"2024-02-17T19:23:01.351743Z","iopub.status.idle":"2024-02-17T19:23:02.288276Z","shell.execute_reply.started":"2024-02-17T19:23:01.351713Z","shell.execute_reply":"2024-02-17T19:23:02.286062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-5'>4.5 train_credit_bureau_a_1_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 2\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:08.649865Z","iopub.execute_input":"2024-02-17T19:23:08.650257Z","iopub.status.idle":"2024-02-17T19:23:12.470646Z","shell.execute_reply.started":"2024-02-17T19:23:08.650229Z","shell.execute_reply":"2024-02-17T19:23:12.469643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 3\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:17.644654Z","iopub.execute_input":"2024-02-17T19:23:17.645177Z","iopub.status.idle":"2024-02-17T19:23:22.816092Z","shell.execute_reply.started":"2024-02-17T19:23:17.645136Z","shell.execute_reply":"2024-02-17T19:23:22.814971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 4\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:23.537543Z","iopub.execute_input":"2024-02-17T19:23:23.538634Z","iopub.status.idle":"2024-02-17T19:23:26.849078Z","shell.execute_reply.started":"2024-02-17T19:23:23.538584Z","shell.execute_reply":"2024-02-17T19:23:26.847983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 5\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:42.615447Z","iopub.execute_input":"2024-02-17T19:23:42.615872Z","iopub.status.idle":"2024-02-17T19:23:44.467627Z","shell.execute_reply.started":"2024-02-17T19:23:42.615839Z","shell.execute_reply":"2024-02-17T19:23:44.466813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-6'>4.6 train_credit_bureau_b_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 6\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:53.653426Z","iopub.execute_input":"2024-02-17T19:23:53.653830Z","iopub.status.idle":"2024-02-17T19:23:53.715169Z","shell.execute_reply.started":"2024-02-17T19:23:53.653801Z","shell.execute_reply":"2024-02-17T19:23:53.714189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-7'>4.7 train_debitcard_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 7\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:23:59.607276Z","iopub.execute_input":"2024-02-17T19:23:59.608641Z","iopub.status.idle":"2024-02-17T19:23:59.638559Z","shell.execute_reply.started":"2024-02-17T19:23:59.608589Z","shell.execute_reply":"2024-02-17T19:23:59.637376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-8'>4.8 train_deposit_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 8\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:05.713709Z","iopub.execute_input":"2024-02-17T19:24:05.714145Z","iopub.status.idle":"2024-02-17T19:24:05.739528Z","shell.execute_reply.started":"2024-02-17T19:24:05.714113Z","shell.execute_reply":"2024-02-17T19:24:05.738324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-9'>4.9 train_other_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 9\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:11.589636Z","iopub.execute_input":"2024-02-17T19:24:11.590163Z","iopub.status.idle":"2024-02-17T19:24:11.614985Z","shell.execute_reply.started":"2024-02-17T19:24:11.590123Z","shell.execute_reply":"2024-02-17T19:24:11.614047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-10'>4.10 train_person_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 10\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:17.933977Z","iopub.execute_input":"2024-02-17T19:24:17.935023Z","iopub.status.idle":"2024-02-17T19:24:18.744645Z","shell.execute_reply.started":"2024-02-17T19:24:17.934985Z","shell.execute_reply":"2024-02-17T19:24:18.743501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-11'>4.11 train_tax_registry_a_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 11\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:25.078840Z","iopub.execute_input":"2024-02-17T19:24:25.079397Z","iopub.status.idle":"2024-02-17T19:24:25.186954Z","shell.execute_reply.started":"2024-02-17T19:24:25.079356Z","shell.execute_reply":"2024-02-17T19:24:25.185927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-12'>4.12 train_tax_registry_b_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 12\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:30.747505Z","iopub.execute_input":"2024-02-17T19:24:30.747923Z","iopub.status.idle":"2024-02-17T19:24:30.790604Z","shell.execute_reply.started":"2024-02-17T19:24:30.747891Z","shell.execute_reply":"2024-02-17T19:24:30.789462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-13'>4.13 train_tax_registry_c_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 13\nprint()\nprint(level_1_files[i])\nprint()\ntotal = file_dict[level_1_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_1_files[i]].isnull().sum()/file_dict[level_1_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:36.921074Z","iopub.execute_input":"2024-02-17T19:24:36.921458Z","iopub.status.idle":"2024-02-17T19:24:37.016831Z","shell.execute_reply.started":"2024-02-17T19:24:36.921430Z","shell.execute_reply":"2024-02-17T19:24:37.015609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-14'>4.14 train_applprev_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:43.802671Z","iopub.execute_input":"2024-02-17T19:24:43.803234Z","iopub.status.idle":"2024-02-17T19:24:44.166889Z","shell.execute_reply.started":"2024-02-17T19:24:43.803189Z","shell.execute_reply":"2024-02-17T19:24:44.164329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-15'>4.15 train_credit_bureau_a_2_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 1\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:49.240850Z","iopub.execute_input":"2024-02-17T19:24:49.241507Z","iopub.status.idle":"2024-02-17T19:24:49.977223Z","shell.execute_reply.started":"2024-02-17T19:24:49.241471Z","shell.execute_reply":"2024-02-17T19:24:49.975972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 2\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:24:55.010961Z","iopub.execute_input":"2024-02-17T19:24:55.011497Z","iopub.status.idle":"2024-02-17T19:24:56.266300Z","shell.execute_reply.started":"2024-02-17T19:24:55.011454Z","shell.execute_reply":"2024-02-17T19:24:56.265050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 3\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:03.207842Z","iopub.execute_input":"2024-02-17T19:25:03.208355Z","iopub.status.idle":"2024-02-17T19:25:03.935701Z","shell.execute_reply.started":"2024-02-17T19:25:03.208318Z","shell.execute_reply":"2024-02-17T19:25:03.934420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 4\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:08.456184Z","iopub.execute_input":"2024-02-17T19:25:08.456570Z","iopub.status.idle":"2024-02-17T19:25:10.879011Z","shell.execute_reply.started":"2024-02-17T19:25:08.456542Z","shell.execute_reply":"2024-02-17T19:25:10.877629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 5\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:15.291230Z","iopub.execute_input":"2024-02-17T19:25:15.291778Z","iopub.status.idle":"2024-02-17T19:25:19.224106Z","shell.execute_reply.started":"2024-02-17T19:25:15.291734Z","shell.execute_reply":"2024-02-17T19:25:19.223018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 6\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:24.209556Z","iopub.execute_input":"2024-02-17T19:25:24.209978Z","iopub.status.idle":"2024-02-17T19:25:27.798825Z","shell.execute_reply.started":"2024-02-17T19:25:24.209923Z","shell.execute_reply":"2024-02-17T19:25:27.797247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 7\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:32.924926Z","iopub.execute_input":"2024-02-17T19:25:32.926320Z","iopub.status.idle":"2024-02-17T19:25:37.476123Z","shell.execute_reply.started":"2024-02-17T19:25:32.926266Z","shell.execute_reply":"2024-02-17T19:25:37.474925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 8\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:43.477261Z","iopub.execute_input":"2024-02-17T19:25:43.477894Z","iopub.status.idle":"2024-02-17T19:25:46.853973Z","shell.execute_reply.started":"2024-02-17T19:25:43.477854Z","shell.execute_reply":"2024-02-17T19:25:46.852949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 9\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:52.047252Z","iopub.execute_input":"2024-02-17T19:25:52.047614Z","iopub.status.idle":"2024-02-17T19:25:53.184467Z","shell.execute_reply.started":"2024-02-17T19:25:52.047588Z","shell.execute_reply":"2024-02-17T19:25:53.183263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 10\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:25:58.027697Z","iopub.execute_input":"2024-02-17T19:25:58.028119Z","iopub.status.idle":"2024-02-17T19:25:59.838967Z","shell.execute_reply.started":"2024-02-17T19:25:58.028087Z","shell.execute_reply":"2024-02-17T19:25:59.837806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 11\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:26:09.719424Z","iopub.execute_input":"2024-02-17T19:26:09.719976Z","iopub.status.idle":"2024-02-17T19:26:12.146716Z","shell.execute_reply.started":"2024-02-17T19:26:09.719918Z","shell.execute_reply":"2024-02-17T19:26:12.145359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-16'>4.16 train_credit_bureau_b_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 12\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:26:19.302146Z","iopub.execute_input":"2024-02-17T19:26:19.302694Z","iopub.status.idle":"2024-02-17T19:26:19.351189Z","shell.execute_reply.started":"2024-02-17T19:26:19.302641Z","shell.execute_reply":"2024-02-17T19:26:19.349901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='4-17'>4.17 train_person_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 13\nprint()\nprint(level_2_files[i])\nprint()\ntotal = file_dict[level_2_files[i]].isnull().sum().sort_values(ascending = False)\npercent = (file_dict[level_2_files[i]].isnull().sum()/file_dict[level_2_files[i]].isnull().count()*100).sort_values(ascending = False)\nmissing_application_train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_application_train_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:26:30.187425Z","iopub.execute_input":"2024-02-17T19:26:30.187833Z","iopub.status.idle":"2024-02-17T19:26:30.270318Z","shell.execute_reply.started":"2024-02-17T19:26:30.187802Z","shell.execute_reply":"2024-02-17T19:26:30.268665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # <a id='5'>5. Data Exploration</a>","metadata":{}},{"cell_type":"code","source":"feature_definition = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv\")\ndef plot_category_distribution(df, target_column_name):\n    temp = df[target_column_name].value_counts()\n    temp_y0 = []\n    temp_y1 = []\n    for val in temp.index:\n        temp_y1.append(np.sum(df[\"target\"][df[target_column_name]==val] == 1))\n        temp_y0.append(np.sum(df[\"target\"][df[target_column_name]==val] == 0))    \n    trace1 = go.Bar(\n        x = temp.index,\n        y = (temp_y1 / temp.sum()) * 100,\n        name='YES'\n    )\n    trace2 = go.Bar(\n        x = temp.index,\n        y = (temp_y0 / temp.sum()) * 100, \n        name='NO'\n    )\n\n    data = [trace1, trace2]\n    try:\n        the_title = f\"Distribution of {target_column_name} ({feature_definition[feature_definition['Variable'] == target_column_name].Description.values[0]}) in terms of loan is default or not in %\"\n    except:\n        the_title = f\"Distribution of {target_column_name}\"\n    layout = go.Layout(\n        title = the_title,\n        width = 1000,\n        xaxis=dict(\n            title=f'Name of {target_column_name}',\n            tickfont=dict(\n                size=14,\n                color='rgb(107, 107, 107)'\n            )\n        ),\n        yaxis=dict(\n            title='Count in %',\n            titlefont=dict(\n                size=16,\n                color='rgb(107, 107, 107)'\n            ),\n            tickfont=dict(\n                size=14,\n                color='rgb(107, 107, 107)'\n            )\n    )\n    )\n\n    fig = go.Figure(data=data, layout=layout)\n    iplot(fig)\n\ndef plot_continues_value_dist(df, column_name):\n    try:\n        print(f\"FSC-{column_name}\")\n        if df[column_name].dtype == \"category\":\n            if len(df[column_name].unique()) > 400:\n                return\n            # Define the default category\n            default_category = \"Not Available\"\n\n            # Get the current categories and add the default category if it's not already present\n            current_categories = df[column_name].cat.categories\n            if default_category not in current_categories:\n                new_categories = [*current_categories, default_category]\n                df[column_name] = df[column_name].cat.set_categories(new_categories)\n            df[column_name].fillna(default_category, inplace=True)\n            plot_category_distribution(df, column_name)\n            return \n        default = df[column_name][df[\"target\"] == 1].dropna()\n        not_default = df[column_name][df[\"target\"] == 0].dropna()\n\n        plt.style.use('_mpl-gallery')\n\n        # plot:\n        fig, ax = plt.subplots(1, 4, figsize=(12, 3))  # 1 row, 2 columns\n\n        fig.suptitle(f'{column_name} and left is default\\n({feature_definition[feature_definition[\"Variable\"] == column_name].Description.values[0]})', fontsize=16, y=1.22)\n        vp = ax[0].violinplot(default,\n                           showmeans=True, showmedians=True, showextrema=True)\n        vp['cmedians'].set_color('black')\n        vp1 = ax[1].violinplot(not_default,\n                           showmeans=True, showmedians=True, showextrema=True)\n        vp1['cmedians'].set_color('black')\n\n        ax[2].hist(default, bins=50, linewidth=0.5, edgecolor=\"white\")\n\n        ax[3].hist(not_default, bins=50, linewidth=0.5, edgecolor=\"white\")\n\n        plt.show()\n    except:\n        print(f\"There is error in column: {column_name}\\n\\n\")\n        ","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:11:31.542235Z","iopub.execute_input":"2024-02-22T00:11:31.542817Z","iopub.status.idle":"2024-02-22T00:11:31.573041Z","shell.execute_reply.started":"2024-02-22T00:11:31.542784Z","shell.execute_reply":"2024-02-22T00:11:31.571886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-1'>5.1 train_base.parquet</a>","metadata":{}},{"cell_type":"code","source":"t = pd.read_csv(f'/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_base.csv')\nprint(f\"From {file_dict[level_0_files[0]].date_decision[0]} To {file_dict[level_0_files[0]].date_decision[len(file_dict[level_0_files[0]].date_decision) - 1]}\" )\nprint(f\"The test data is From {t.date_decision[0]} \" )","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:27:41.001022Z","iopub.execute_input":"2024-02-17T19:27:41.004278Z","iopub.status.idle":"2024-02-17T19:27:41.029678Z","shell.execute_reply.started":"2024-02-17T19:27:41.004057Z","shell.execute_reply":"2024-02-17T19:27:41.027763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.barh([\"default\", \"good\"], [(file_dict[level_0_files[0]][\"target\"]==1).sum(),(file_dict[level_0_files[0]][\"target\"]==0).sum()], align='center')","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:27:45.149003Z","iopub.execute_input":"2024-02-17T19:27:45.152756Z","iopub.status.idle":"2024-02-17T19:27:45.393807Z","shell.execute_reply.started":"2024-02-17T19:27:45.152617Z","shell.execute_reply":"2024-02-17T19:27:45.391791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\ntarget_column_name = \"WEEK_NUM\"\nplot_category_distribution(file_dict[level_0_files[i]], target_column_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:29:10.150122Z","iopub.execute_input":"2024-02-17T19:29:10.150558Z","iopub.status.idle":"2024-02-17T19:29:10.522288Z","shell.execute_reply.started":"2024-02-17T19:29:10.150526Z","shell.execute_reply":"2024-02-17T19:29:10.521039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-2'>5.2 train_static_0_0.parquet</a>","metadata":{}},{"cell_type":"code","source":"i = 1\njoined_train_static_0_0 = file_dict[level_0_files[0]].join(file_dict[level_0_files[i]].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_0\")","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:29:35.322762Z","iopub.execute_input":"2024-02-17T19:29:35.323891Z","iopub.status.idle":"2024-02-17T19:29:39.616831Z","shell.execute_reply.started":"2024-02-17T19:29:35.323852Z","shell.execute_reply":"2024-02-17T19:29:39.615575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nfor c_name in file_dict[level_0_files[i]].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined_train_static_0_0, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:30:48.675439Z","iopub.execute_input":"2024-02-17T19:30:48.675915Z","iopub.status.idle":"2024-02-17T19:37:40.788681Z","shell.execute_reply.started":"2024-02-17T19:30:48.675853Z","shell.execute_reply":"2024-02-17T19:37:40.787411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-3'>5.3 train_static_cb_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 3\nbase = level_0_files[0]\ntarget_df = level_0_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_0\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:39:04.077794Z","iopub.execute_input":"2024-02-17T19:39:04.079041Z","iopub.status.idle":"2024-02-17T19:40:38.523150Z","shell.execute_reply.started":"2024-02-17T19:39:04.078999Z","shell.execute_reply":"2024-02-17T19:40:38.522028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_dict[target_df].head()","metadata":{"execution":{"iopub.status.busy":"2024-02-22T00:14:56.015159Z","iopub.execute_input":"2024-02-22T00:14:56.015593Z","iopub.status.idle":"2024-02-22T00:14:56.046585Z","shell.execute_reply.started":"2024-02-22T00:14:56.015559Z","shell.execute_reply":"2024-02-22T00:14:56.045232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-4'>5.4 train_applprev_1_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:41:00.038606Z","iopub.execute_input":"2024-02-17T19:41:00.039092Z","iopub.status.idle":"2024-02-17T19:42:38.385350Z","shell.execute_reply.started":"2024-02-17T19:41:00.039055Z","shell.execute_reply":"2024-02-17T19:42:38.384025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-5'>5.5 train_credit_bureau_a_1_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 2\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:42:51.709109Z","iopub.execute_input":"2024-02-17T19:42:51.709517Z","iopub.status.idle":"2024-02-17T19:44:24.179196Z","shell.execute_reply.started":"2024-02-17T19:42:51.709486Z","shell.execute_reply":"2024-02-17T19:44:24.177763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-6'>5.6 train_credit_bureau_b_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 6\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:45:02.148456Z","iopub.execute_input":"2024-02-17T19:45:02.148903Z","iopub.status.idle":"2024-02-17T19:45:36.247863Z","shell.execute_reply.started":"2024-02-17T19:45:02.148872Z","shell.execute_reply":"2024-02-17T19:45:36.246555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-7'>5.7 train_debitcard_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 7\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:46:14.363407Z","iopub.execute_input":"2024-02-17T19:46:14.363839Z","iopub.status.idle":"2024-02-17T19:46:17.875700Z","shell.execute_reply.started":"2024-02-17T19:46:14.363805Z","shell.execute_reply":"2024-02-17T19:46:17.874344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-8'>5.8 train_deposit_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 8\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:46:45.604240Z","iopub.execute_input":"2024-02-17T19:46:45.604799Z","iopub.status.idle":"2024-02-17T19:46:47.909758Z","shell.execute_reply.started":"2024-02-17T19:46:45.604761Z","shell.execute_reply":"2024-02-17T19:46:47.908001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" ### <a id='5-9'>5.9 train_other_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 9\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:47:00.461419Z","iopub.execute_input":"2024-02-17T19:47:00.461894Z","iopub.status.idle":"2024-02-17T19:47:06.855856Z","shell.execute_reply.started":"2024-02-17T19:47:00.461856Z","shell.execute_reply":"2024-02-17T19:47:06.854643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-10'>5.10 train_person_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 10\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:47:18.368140Z","iopub.execute_input":"2024-02-17T19:47:18.368561Z","iopub.status.idle":"2024-02-17T19:47:50.822562Z","shell.execute_reply.started":"2024-02-17T19:47:18.368527Z","shell.execute_reply":"2024-02-17T19:47:50.821329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-11'>5.11 train_tax_registry_a_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 11\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in [\"amount_4527230A\"]:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:47:50.824909Z","iopub.execute_input":"2024-02-17T19:47:50.825292Z","iopub.status.idle":"2024-02-17T19:47:59.766237Z","shell.execute_reply.started":"2024-02-17T19:47:50.825259Z","shell.execute_reply":"2024-02-17T19:47:59.764777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-12'>5.12 train_tax_registry_b_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 12\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in [\"amount_4917619A\"]:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:48:06.339825Z","iopub.execute_input":"2024-02-17T19:48:06.340228Z","iopub.status.idle":"2024-02-17T19:48:10.316125Z","shell.execute_reply.started":"2024-02-17T19:48:06.340199Z","shell.execute_reply":"2024-02-17T19:48:10.314720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-13'>5.13 train_tax_registry_c_1.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 13\nbase = level_0_files[0]\ntarget_df = level_1_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in [\"pmtamount_36A\"]:\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:48:14.521621Z","iopub.execute_input":"2024-02-17T19:48:14.522021Z","iopub.status.idle":"2024-02-17T19:48:22.815232Z","shell.execute_reply.started":"2024-02-17T19:48:14.521992Z","shell.execute_reply":"2024-02-17T19:48:22.814043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-14'>5.14 train_applprev_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 0\nbase = level_0_files[0]\ntarget_df = level_2_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_2\")\nfor c_name in file_dict[target_df].columns.values:\n    if file_dict[target_df].columns.dtype == \"category\":\n        if len(file_dict[target_df].unique()) > 500:\n            continue\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:48:22.948969Z","iopub.execute_input":"2024-02-17T19:48:22.950090Z","iopub.status.idle":"2024-02-17T19:48:27.990563Z","shell.execute_reply.started":"2024-02-17T19:48:22.950038Z","shell.execute_reply":"2024-02-17T19:48:27.989333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-15'>5.15 train_credit_bureau_a_2_0.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 1\nbase = level_0_files[0]\ntarget_df = level_2_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_2\")\nfor c_name in file_dict[target_df].columns.values:\n    if file_dict[target_df].columns.dtype == \"category\":\n        if len(file_dict[target_df].unique()) > 500:\n            continue\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:48:37.329959Z","iopub.execute_input":"2024-02-17T19:48:37.330371Z","iopub.status.idle":"2024-02-17T19:49:10.115326Z","shell.execute_reply.started":"2024-02-17T19:48:37.330338Z","shell.execute_reply":"2024-02-17T19:49:10.113776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-16'>5.16 train_credit_bureau_b_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 12\nbase = level_0_files[0]\ntarget_df = level_2_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_1\")\nfor c_name in file_dict[target_df].columns.values:\n    if file_dict[target_df].columns.dtype == \"category\":\n        if len(file_dict[target_df].unique()) > 500:\n            continue\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:49:10.117996Z","iopub.execute_input":"2024-02-17T19:49:10.118490Z","iopub.status.idle":"2024-02-17T19:49:20.355223Z","shell.execute_reply.started":"2024-02-17T19:49:10.118444Z","shell.execute_reply":"2024-02-17T19:49:20.354230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <a id='5-17'>5.17 train_person_2.csv</a>","metadata":{}},{"cell_type":"code","source":"i = 13\nbase = level_0_files[0]\ntarget_df = level_2_files[i]\nprint(target_df)\nprint()\njoined = file_dict[base].join(file_dict[target_df].set_index('case_id'), how=\"left\", on=\"case_id\", rsuffix=\"_2\")\nfor c_name in file_dict[target_df].columns.values:\n    if file_dict[target_df].columns.dtype == \"category\":\n        if len(file_dict[target_df].unique()) > 500:\n            continue\n    if c_name == \"case_id\":\n        continue\n    plot_continues_value_dist(joined, c_name)","metadata":{"execution":{"iopub.status.busy":"2024-02-17T19:49:28.700109Z","iopub.execute_input":"2024-02-17T19:49:28.700513Z","iopub.status.idle":"2024-02-17T19:49:30.875970Z","shell.execute_reply.started":"2024-02-17T19:49:28.700483Z","shell.execute_reply":"2024-02-17T19:49:30.874812Z"},"trusted":true},"execution_count":null,"outputs":[]}]}