{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from IPython.display import HTML, Markdown\nimport time\n\nhandle = display(HTML(\"\"\"\"\"\"), display_id='html_marquee1')\ntime.sleep(2)\nhandle = display(HTML(\"\"\"\"\"\"), display_id='html_marquee1', update=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:31:42.241557Z","iopub.execute_input":"2024-02-24T13:31:42.242510Z","iopub.status.idle":"2024-02-24T13:31:44.286395Z","shell.execute_reply.started":"2024-02-24T13:31:42.242444Z","shell.execute_reply":"2024-02-24T13:31:44.285466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Executive Summary
\n\nThis notebook is intended to draw comprehensive insights on the dataset of **Home Credit - Credit Risk Model Stability** competition at Kaggle.\n\n \n✨ Reader Note: The status of this notebook is 'Work-in-Progress'. You can expect to see more EDA insights in the future versions.\n
","metadata":{}},{"cell_type":"code","source":"!pip install duckdb --quiet","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-24T13:31:44.288647Z","iopub.execute_input":"2024-02-24T13:31:44.289106Z","iopub.status.idle":"2024-02-24T13:32:02.149326Z","shell.execute_reply.started":"2024-02-24T13:31:44.289046Z","shell.execute_reply":"2024-02-24T13:32:02.147746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport pandas as pd\nimport numpy as np\nimport datetime as dt\n\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport seaborn as sns\nimport missingno as msno\n\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\nimport plotly.offline\n\nfrom colorama import Fore, Style, init\nfrom pprint import pprint\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:02.151690Z","iopub.execute_input":"2024-02-24T13:32:02.152177Z","iopub.status.idle":"2024-02-24T13:32:04.090052Z","shell.execute_reply.started":"2024-02-24T13:32:02.152131Z","shell.execute_reply":"2024-02-24T13:32:04.088904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import duckdb\n\ncon = duckdb.connect()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:04.092577Z","iopub.execute_input":"2024-02-24T13:32:04.093106Z","iopub.status.idle":"2024-02-24T13:32:04.138592Z","shell.execute_reply.started":"2024-02-24T13:32:04.093073Z","shell.execute_reply":"2024-02-24T13:32:04.137595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reused from https://stackoverflow.com/questions/41125909/find-elements-in-one-list-that-are-not-in-the-other\ndef setdiff_sorted(array1,array2,assume_unique=False):\n ans = np.setdiff1d(array1,array2,assume_unique).tolist()\n if assume_unique:\n return sorted(ans)\n return ans\n\n# Color printing\n# inspired by https://www.kaggle.com/code/ravi20076/sleepstate-eda-baseline\ndef PrintColor(text:str, color = Fore.BLUE, style = Style.BRIGHT):\n \"Prints color outputs using colorama using a text F-string\";\n print(style + color + text + Style.RESET_ALL);\n \n# reused from https://www.kaggle.com/code/gvyshnya/to-sleep-or-not-to-sleep-deep-eda-dive\ndef summarize_dataframe(df):\n summary_df = pd.DataFrame(df.dtypes, columns=['dtypes'])\n summary_df['missing#'] = df.isna().sum().values*100\n summary_df['missing%'] = (df.isna().sum().values*100)/len(df)\n summary_df['uniques'] = df.nunique().values\n summary_df['first_value'] = df.iloc[0].values\n summary_df['last_value'] = df.iloc[len(df)-1].values\n summary_df['count'] = df.count().values\n #sum['skew'] = df.skew().values\n desc = pd.DataFrame(df.describe().T)\n summary_df['min'] = desc['min']\n summary_df['max'] = desc['max']\n summary_df['mean'] = desc['mean']\n return summary_df\n\n\n# reused from https://www.kaggle.com/code/gvyshnya/to-sleep-or-not-to-sleep-deep-eda-dive\nfrom pandas.api.types import is_datetime64_ns_dtype\nfrom pandas.api.types import is_datetime64_any_dtype\ndef reduce_mem_usage(df):\n \"\"\" iterate through all numeric columns of a dataframe and modify the data type\n to reduce memory usage. \n \"\"\"\n start_mem = df.memory_usage().sum() / 1024**2\n print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n \n for col in df.columns:\n col_type = df[col].dtype\n\n # is_datetime64_any_dtype is more generic then is_datetime64_ns_dtype\n if col_type != object and not is_datetime64_any_dtype(df[col]):\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float16)\n\n end_mem = df.memory_usage().sum() / 1024**2\n print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n decrease = 100 * (start_mem - end_mem) / start_mem\n print(f'Decreased by {decrease:.2f}%')\n \n return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:04.140747Z","iopub.execute_input":"2024-02-24T13:32:04.141238Z","iopub.status.idle":"2024-02-24T13:32:04.172776Z","shell.execute_reply.started":"2024-02-24T13:32:04.141192Z","shell.execute_reply":"2024-02-24T13:32:04.171162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Description
\n\nIn this competition, we are going to predict default of clients based on internal and external information that are available for each client. Scoring is performed using custom metric that not only evaluates the AUC of predictions but also considers the stability of predictions model across the data range of the test set. \n\nTo better understand this metric, please refer to the [Evaluation](https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/overview/evaluation) page in the contest documentation.\n\nAs mentioned in the competition [documnetation](https://www.kaggle.com/competitions/home-credit-credit-risk-model-stability/data), this dataset contains a large number of tables as a result of utilizing diverse data sources and the varying levels of data aggregation used while preparing the dataset. \n\n ✅ Note: All files listed below are found in both .csv and .parquet formats.
\n\nFirst of all, let's review the inventory of the files in the dataset.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n PrintColor(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:04.178839Z","iopub.execute_input":"2024-02-24T13:32:04.180189Z","iopub.status.idle":"2024-02-24T13:32:04.219923Z","shell.execute_reply.started":"2024-02-24T13:32:04.180130Z","shell.execute_reply":"2024-02-24T13:32:04.218602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's extract the feature descriptions provided as a metadata asset in the dataset","metadata":{}},{"cell_type":"code","source":"feature_descr_path = '/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv'\n\ndf = pd.read_csv(feature_descr_path, delimiter=',')\n\ndf.style.set_caption(\"Feature descriptions\"). \\\nset_properties(**{'border': '1.3px solid blue',\n 'color': 'grey'})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:04.221745Z","iopub.execute_input":"2024-02-24T13:32:04.222466Z","iopub.status.idle":"2024-02-24T13:32:04.430691Z","shell.execute_reply.started":"2024-02-24T13:32:04.222422Z","shell.execute_reply":"2024-02-24T13:32:04.429618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Inspection: Static 0 data
\n\nAs mentioned in the dataset description, these is the internal data source with the properties of level 0.\n\nThey are represented by certain train- and test-set files. Train Files are listed below:\n\n- *train_static_0_0.csv*\n- *train_static_0_1.csv*\n\nTest Files are listed below:\n\n- *test_static_0_0.csv*\n- *test_static_0_1.csv*\n- *test_static_0_2.csv*\n\n## Static 0 Data: Training set inspection
","metadata":{}},{"cell_type":"code","source":"train_static_0 = '/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_0.csv'\ntrain_static_1 = '/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_1.csv'\n\ncon.execute(f\"\"\"\n CREATE TABLE train_static_0 AS\n SELECT * \n FROM read_csv('{train_static_0}', AUTO_DETECT=TRUE);\n\"\"\")\n\ncon.execute(f\"\"\"\n CREATE TABLE train_static_1 AS\n SELECT * \n FROM read_csv('{train_static_1}', AUTO_DETECT=TRUE);\n\"\"\")","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:04.431841Z","iopub.execute_input":"2024-02-24T13:32:04.432389Z","iopub.status.idle":"2024-02-24T13:32:29.441991Z","shell.execute_reply.started":"2024-02-24T13:32:04.432353Z","shell.execute_reply":"2024-02-24T13:32:29.440834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_boxplot_data(col_name, where_clause = None):\n min_max_sql = \"\"\n quantile_sql = \"\"\n if where_clause:\n min_max_sql = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n )\n SELECT '{col_name}' AS attribute, \n MIN({col_name}) AS min_value, \n AVG({col_name}) AS mean_value, \n MAX({col_name}) AS max_value,\n stddev_pop({col_name}) AS stddev\n FROM table_union\n WHERE {where_clause}\n \"\"\"\n quantile_sql = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n ), \n stats as (\n SELECT\n percentile_disc([0.25, 0.50, 0.75]) WITHIN GROUP \n (ORDER BY {col_name}) AS percentiles\n FROM table_union\n WHERE {where_clause}\n )\n SELECT\n '{col_name}' AS attribute,\n percentiles[1] as q1,\n percentiles[2] as median,\n percentiles[3] as q3\n FROM stats;\n \"\"\"\n else:\n min_max_sql = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n )\n SELECT '{col_name}' AS attribute, \n MIN({col_name}) AS min_value, \n AVG({col_name}) AS mean_value, \n MAX({col_name}) AS max_value,\n stddev_pop({col_name}) AS stddev\n FROM table_union\n \"\"\"\n quantile_sql = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n ), \n stats as (\n SELECT\n percentile_disc([0.25, 0.50, 0.75]) WITHIN GROUP \n (ORDER BY {col_name}) AS percentiles\n FROM table_union\n )\n SELECT\n '{col_name}' AS attribute,\n percentiles[1] as q1,\n percentiles[2] as median,\n percentiles[3] as q3\n FROM stats;\n \"\"\"\n df_mean_max = con.execute(min_max_sql).fetchdf()\n df_quantile = con.execute(quantile_sql).fetchdf()\n \n df = pd.merge(\n df_mean_max,\n df_quantile,\n how=\"inner\",\n on='attribute')\n \n #rearrange the order of cols to make it a canonical boxplot style\n df = df[['attribute', 'min_value', 'q1', 'median', 'q3', 'max_value', 'mean_value', 'stddev']]\n \n return df\n\ndef build_boxplot(col_name, where_clause = None):\n df = get_boxplot_data(col_name, where_clause)\n fig_title = \"\"\n \n bar_name = \"\".join([col_name])\n \n if where_clause:\n fig_title = \"\".join([col_name, \": \", where_clause])\n else:\n fig_title = \"\".join([col_name, \": Entire population\"])\n \n fig = go.Figure()\n\n fig.add_trace(go.Box(\n y= [ df['min_value'].iloc[0], df['max_value'].iloc[0] ],\n name=bar_name, boxpoints=False,)\n )\n\n fig.update_traces(q1=[ df['q1'].iloc[0] ], median=[ df['median'].iloc[0] ],\n q3=[ df['q3'].iloc[0] ], lowerfence=[df['min_value'].iloc[0]],\n upperfence=[df['max_value'].iloc[0]], mean=[ df['mean_value'].iloc[0] ],\n sd=[ df['stddev'].iloc[0]]\n )\n # Update visual layout\n fig.update_layout(\n showlegend=False,\n width=600,\n height=400,\n autosize=False,\n margin=dict(t=15, b=0, l=5, r=5),\n title=dict(text=fig_title, font=dict(size=16), automargin=True, yref='paper'),\n title_x=0.5, # center the title\n template=\"plotly_white\",\n colorway=px.colors.qualitative.Prism ,\n )\n \n # update font size at the axes\n fig.update_coloraxes(colorbar_tickfont_size=10)\n # Update font in the titles: Apparently subplot titles are annotations (Subplot font size is hardcoded to 16pt · Issue #985)\n fig.update_annotations(font_size=12)\n # Reduce opacity\n fig.update_traces(opacity=0.75)\n \n return fig","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:29.445224Z","iopub.execute_input":"2024-02-24T13:32:29.445737Z","iopub.status.idle":"2024-02-24T13:32:29.465901Z","shell.execute_reply.started":"2024-02-24T13:32:29.445691Z","shell.execute_reply":"2024-02-24T13:32:29.464723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://duckdb.org/docs/sql/aggregates\ndef get_histogram_data(col_name, where_clause = None):\n query = \"\"\n if where_clause:\n query = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n )\n SELECT histogram({col_name}) FROM table_union\n WHERE {where_clause};\n \"\"\"\n else:\n query = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n )\n SELECT histogram({col_name}) FROM table_union;\n \"\"\"\n df = con.execute(query).fetchdf()\n \n hist_col_name = f\"histogram({col_name})\"\n\n histogram_dict = df[hist_col_name].iloc[0] # dictionary: dict_keys(['key', 'value'])\n\n frame = {\n col_name: histogram_dict.get('key'),\n 'record_count': histogram_dict.get('value')}\n \n # Creating DataFrame by passing Dictionary\n agg_data = pd.DataFrame(frame)\n return agg_data\n \ndef build_histogram(col_name, where_clause = None):\n figure_title = \"\"\n if where_clause:\n figure_title = f\"Distribution of {col_name}; filter: {where_clause}\"\n else:\n figure_title = f\"Distribution of {col_name}\"\n \n agg_data = get_histogram_data(col_name, where_clause)\n \n fig = px.histogram(agg_data, x=col_name, marginal=\"rug\",\n title=figure_title,\n color_discrete_sequence=px.colors.qualitative.Prism)\n return fig","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:29.471798Z","iopub.execute_input":"2024-02-24T13:32:29.472199Z","iopub.status.idle":"2024-02-24T13:32:29.482615Z","shell.execute_reply.started":"2024-02-24T13:32:29.472157Z","shell.execute_reply":"2024-02-24T13:32:29.481450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_histogram_data_2(col_name, multiplier=1, where_clause = None):\n query = \"\"\n if where_clause:\n query = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n )\n SELECT FLOOR({col_name}/{multiplier}.00)*{multiplier} AS {col_name}, \n COUNT(*) AS record_count\n FROM table_union\n WHERE {where_clause}\n GROUP BY FLOOR({col_name}/{multiplier}.00)*{multiplier}\n ORDER BY 1\n \"\"\"\n else:\n query = f\"\"\"\n WITH table_union as (\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n )\n SELECT FLOOR({col_name}/{multiplier}.00)*{multiplier} AS {col_name}, \n COUNT(*) AS record_count\n FROM table_union\n GROUP BY FLOOR({col_name}/{multiplier}.00)*{multiplier}\n ORDER BY 1\n \"\"\"\n df_agg = con.execute(query).fetchdf()\n return df_agg\n\ndef build_histogram_2(col_name, multiplier=1, where_clause = None):\n fig_title = \"\"\n if where_clause:\n fig_title = f\"Distribution of {col_name}; filtered: {where_clause}\"\n else:\n fig_title = f\"Distribution of {col_name}\"\n df_agg = get_histogram_data_2(col_name, multiplier, where_clause)\n fig = px.bar(df_agg, x=col_name, y=\"record_count\",\n title=fig_title,\n color_discrete_sequence=px.colors.qualitative.Prism)\n return fig","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:29.484577Z","iopub.execute_input":"2024-02-24T13:32:29.485046Z","iopub.status.idle":"2024-02-24T13:32:29.499906Z","shell.execute_reply.started":"2024-02-24T13:32:29.484998Z","shell.execute_reply":"2024-02-24T13:32:29.498699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_static_0_data_training():\n \n sql = f\"\"\"\n SELECT\n *\n FROM train_static_0\n UNION ALL\n SELECT \n *\n FROM train_static_1\n \"\"\"\n df = con.execute(sql).fetchdf()\n \n return df\n\nstatic_0_df = get_static_0_data_training()\n\nstatic_0_df = reduce_mem_usage(static_0_df)\n\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:29.501122Z","iopub.execute_input":"2024-02-24T13:32:29.501594Z","iopub.status.idle":"2024-02-24T13:32:51.556586Z","shell.execute_reply.started":"2024-02-24T13:32:29.501543Z","shell.execute_reply":"2024-02-24T13:32:51.555275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"static_0_df.head().style.set_caption(\"Sample of Static 0 data (training set)\"). \\\nset_properties(**{'border': '1.3px solid blue',\n 'color': 'grey'})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:51.558108Z","iopub.execute_input":"2024-02-24T13:32:51.558528Z","iopub.status.idle":"2024-02-24T13:32:51.650955Z","shell.execute_reply.started":"2024-02-24T13:32:51.558493Z","shell.execute_reply":"2024-02-24T13:32:51.649814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's review the statistical summary of the Static 0 data (training set):","metadata":{}},{"cell_type":"code","source":"summ_train_df = summarize_dataframe(static_0_df)\n\nsumm_train_df.style.set_caption(\"Static 0 Data (training) summary\"). \\\nset_properties(**{'border': '1.3px solid blue',\n 'color': 'grey'})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:32:51.652600Z","iopub.execute_input":"2024-02-24T13:32:51.652947Z","iopub.status.idle":"2024-02-24T13:33:33.683638Z","shell.execute_reply.started":"2024-02-24T13:32:51.652917Z","shell.execute_reply":"2024-02-24T13:33:33.681959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check for the missing values in the static 0 dataset (training set).","metadata":{}},{"cell_type":"code","source":"msno.bar(static_0_df, color=(0.3,0.3,0.5))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:33.685298Z","iopub.execute_input":"2024-02-24T13:33:33.686468Z","iopub.status.idle":"2024-02-24T13:33:44.179425Z","shell.execute_reply.started":"2024-02-24T13:33:33.686392Z","shell.execute_reply":"2024-02-24T13:33:44.177567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's assess the percentage of missing values in each of the columns of Static 0 dataset.\n\nFirst of all, let's extract the list of columns without missing values:","metadata":{}},{"cell_type":"code","source":"filtered_df = summ_train_df[summ_train_df['missing%'] == 0]\ntrain_cols_with_full_data = [col for col in filtered_df.index]\nfor col in train_cols_with_full_data:\n PrintColor(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.181274Z","iopub.execute_input":"2024-02-24T13:33:44.181667Z","iopub.status.idle":"2024-02-24T13:33:44.192729Z","shell.execute_reply.started":"2024-02-24T13:33:44.181634Z","shell.execute_reply":"2024-02-24T13:33:44.191451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The columns above are ideal attributes for the EDA and feature engineering in ML experiments down the road.\n\nNow, let review the features that have some missing values yet the percentage of records with the missing values does not exceed `21%`. Such features will be feasible to use in the ML pipelines down the road after appropriate data imputation.","metadata":{}},{"cell_type":"code","source":"filtered_df = summ_train_df[(summ_train_df['missing%'] > 0) & (summ_train_df['missing%'] < 21)]\ntrain_cols_with_minor_nan = [col for col in filtered_df.index]\nfor col in train_cols_with_minor_nan:\n PrintColor(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.194163Z","iopub.execute_input":"2024-02-24T13:33:44.194616Z","iopub.status.idle":"2024-02-24T13:33:44.207885Z","shell.execute_reply.started":"2024-02-24T13:33:44.194582Z","shell.execute_reply":"2024-02-24T13:33:44.206556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, let's look at the attributes with the inacceptably high ratio of missing values. These are","metadata":{}},{"cell_type":"code","source":"filtered_df = summ_train_df[(summ_train_df['missing%'] >= 21)]\ntrain_cols_with_major_nan = [col for col in filtered_df.index]\nfor col in train_cols_with_major_nan:\n PrintColor(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.209223Z","iopub.execute_input":"2024-02-24T13:33:44.209781Z","iopub.status.idle":"2024-02-24T13:33:44.219111Z","shell.execute_reply.started":"2024-02-24T13:33:44.209741Z","shell.execute_reply":"2024-02-24T13:33:44.217851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The latter list of attributes is not feasible to use in the ML pipelines down the road.","metadata":{}},{"cell_type":"markdown","source":"## Static 0 Data: Test set inspection
","metadata":{}},{"cell_type":"code","source":"test_static_0 = '/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_static_0_0.csv'\ntest_static_1 = '/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_static_0_1.csv'\ntest_static_2 = '/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/test_static_0_2.csv'\n\ncon.execute(f\"\"\"\n CREATE TABLE test_static_0 AS\n SELECT * \n FROM read_csv('{test_static_0}', AUTO_DETECT=TRUE);\n\"\"\")\n\ncon.execute(f\"\"\"\n CREATE TABLE test_static_1 AS\n SELECT * \n FROM read_csv('{test_static_1}', AUTO_DETECT=TRUE);\n\"\"\")\n\ncon.execute(f\"\"\"\n CREATE TABLE test_static_2 AS\n SELECT * \n FROM read_csv('{test_static_2}', AUTO_DETECT=TRUE);\n\"\"\")\n\ndef get_static_0_data_test():\n \n sql = f\"\"\"\n SELECT\n *\n FROM test_static_0\n UNION ALL\n SELECT \n *\n FROM test_static_1\n UNION ALL\n SELECT \n *\n FROM test_static_2\n \"\"\"\n df = con.execute(sql).fetchdf()\n \n return df\n\nstatic_0_test_df = get_static_0_data_test()\n\nstatic_0_test_df = reduce_mem_usage(static_0_test_df)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.221148Z","iopub.execute_input":"2024-02-24T13:33:44.221720Z","iopub.status.idle":"2024-02-24T13:33:44.612718Z","shell.execute_reply.started":"2024-02-24T13:33:44.221663Z","shell.execute_reply":"2024-02-24T13:33:44.611484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"static_0_test_df.head().style.set_caption(\"Sample of Static 0 data (test set)\"). \\\nset_properties(**{'border': '1.3px solid blue',\n 'color': 'grey'})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.614504Z","iopub.execute_input":"2024-02-24T13:33:44.615665Z","iopub.status.idle":"2024-02-24T13:33:44.703174Z","shell.execute_reply.started":"2024-02-24T13:33:44.615625Z","shell.execute_reply":"2024-02-24T13:33:44.702008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's review the summary statistics for Static 0 data (test set):","metadata":{}},{"cell_type":"code","source":"summ_test_df = summarize_dataframe(static_0_test_df)\n\nsumm_test_df.style.set_caption(\"Static 0 Data (test) summary\"). \\\nset_properties(**{'border': '1.3px solid blue',\n 'color': 'grey'})","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.704671Z","iopub.execute_input":"2024-02-24T13:33:44.705010Z","iopub.status.idle":"2024-02-24T13:33:44.970134Z","shell.execute_reply.started":"2024-02-24T13:33:44.704974Z","shell.execute_reply":"2024-02-24T13:33:44.969210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check for the missing values in the static 0 dataset (test set).","metadata":{}},{"cell_type":"code","source":"msno.bar(static_0_test_df, color=(0.3,0.3,0.5))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:44.971291Z","iopub.execute_input":"2024-02-24T13:33:44.972351Z","iopub.status.idle":"2024-02-24T13:33:50.026127Z","shell.execute_reply.started":"2024-02-24T13:33:44.972307Z","shell.execute_reply":"2024-02-24T13:33:50.023857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's assess the percentage of missing values in each of the columns of Static 0 dataset (test set).\n\nFirst of all, let's extract the list of columns without missing values:","metadata":{}},{"cell_type":"code","source":"filtered_df = summ_test_df[summ_test_df['missing%'] == 0]\ntest_cols_with_full_data = [col for col in filtered_df.index]\nfor col in test_cols_with_full_data:\n PrintColor(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.027776Z","iopub.execute_input":"2024-02-24T13:33:50.028226Z","iopub.status.idle":"2024-02-24T13:33:50.038967Z","shell.execute_reply.started":"2024-02-24T13:33:50.028187Z","shell.execute_reply":"2024-02-24T13:33:50.037601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The columns above are ideal attributes for the EDA and feature engineering in ML experiments down the road.\n\nNow, let review the features that have some missing values yet the percentage of records with the missing values does not exceed `21%`. Such features will be feasible to use in the ML pipelines down the road after appropriate data imputation.","metadata":{}},{"cell_type":"code","source":"filtered_df = summ_test_df[(summ_test_df['missing%'] > 0) & (summ_test_df['missing%'] < 21)]\ntest_cols_with_minor_nan = [col for col in filtered_df.index]\nfor col in test_cols_with_minor_nan:\n PrintColor(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.041223Z","iopub.execute_input":"2024-02-24T13:33:50.041702Z","iopub.status.idle":"2024-02-24T13:33:50.059529Z","shell.execute_reply.started":"2024-02-24T13:33:50.041659Z","shell.execute_reply":"2024-02-24T13:33:50.057813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, let's look at the attributes with the inacceptably high ratio of missing values. These are","metadata":{}},{"cell_type":"code","source":"filtered_df = summ_test_df[(summ_test_df['missing%'] >= 21)]\ntest_cols_with_major_nan = [col for col in filtered_df.index]\nfor col in test_cols_with_major_nan:\n PrintColor(col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.063664Z","iopub.execute_input":"2024-02-24T13:33:50.064196Z","iopub.status.idle":"2024-02-24T13:33:50.075619Z","shell.execute_reply.started":"2024-02-24T13:33:50.064138Z","shell.execute_reply":"2024-02-24T13:33:50.074592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Static 0 Data: Training vs. Test set inference
\n\nLet's compare the list of columns without missing value in Static 0 data in training and testing set","metadata":{}},{"cell_type":"code","source":"full_col_diff = setdiff_sorted(train_cols_with_full_data, test_cols_with_full_data,assume_unique=True)\n\nfor col in full_col_diff:\n PrintColor(col)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.076693Z","iopub.execute_input":"2024-02-24T13:33:50.077032Z","iopub.status.idle":"2024-02-24T13:33:50.092732Z","shell.execute_reply.started":"2024-02-24T13:33:50.077002Z","shell.execute_reply":"2024-02-24T13:33:50.091527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, `'deferredmnthsnum_166L'` column that is totally without missing values in the training set, has missing values in the test set. Furthermore, the ratio of the missing values in this column in the training set exceeds `21%` (please see above).\n\n**Inference:** As a matter of the fact, the above-mentioned findings indicate `'deferredmnthsnum_166L'` feature to be excluded from the subset of the features of *Static 0* data to be used in the ML experiments down the road.\n\nNow, let's compare the list of columns with the ratio of missing values below `21%` in the training and test sets. Below, we are going to list the columns with the moderate ratio of *NaN* values in the training set that are not on the same scale of *NaN* values in the test set:","metadata":{}},{"cell_type":"code","source":"minor_nan_col_diff = setdiff_sorted(train_cols_with_minor_nan, test_cols_with_minor_nan,assume_unique=True)\nfor col in minor_nan_col_diff:\n PrintColor(col)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.094724Z","iopub.execute_input":"2024-02-24T13:33:50.095184Z","iopub.status.idle":"2024-02-24T13:33:50.104158Z","shell.execute_reply.started":"2024-02-24T13:33:50.095141Z","shell.execute_reply":"2024-02-24T13:33:50.103003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we are going to check the columns from the list above in the test set. There are two options possible\n\n- the respective column has **zero** missing values in the visible part of the test set; in such a case, we can retain such a column in the subset of parameters for the ML pipelines/experiments down the road;\n- the respective column has **more then `21%`** of missing values in the visible part of the test set; in such a case, we disqualify this attribute as well as exclude it from the ML experiments down the road. \n\n**Inference:** The columns to retain in the dataset for the purpose of ML experiemnts are listed below\n- `annuitynextmonth_57A`\n- `credtype_322L`\n- `currdebt_22A`\n- `currdebtcredtyperange_828A`\n- `inittransactioncode_186L`\n- `numinstls_657L`\n- `totaldebt_9A`\n- `totalsettled_863A`\n- `twobodfilling_608L`\n\nIn turn, the columns to be ultimately disqualified are listed below\n- `lastapplicationdate_877D`\n- `lastst_736L`\n- `opencred_647L`\n- `paytype1st_925L`\n- `paytype_783L`\n- `price_1097A`\n","metadata":{}},{"cell_type":"markdown","source":"Finally, we are going to list the columns with the moderate ratio of *NaN* values in the test set that are not on the same scale of *NaN* values in the training set:","metadata":{}},{"cell_type":"code","source":"minor_nan_col_diff_2 = setdiff_sorted(test_cols_with_minor_nan, train_cols_with_minor_nan, assume_unique=True)\nfor col in minor_nan_col_diff_2:\n PrintColor(col)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.112725Z","iopub.execute_input":"2024-02-24T13:33:50.113619Z","iopub.status.idle":"2024-02-24T13:33:50.119774Z","shell.execute_reply.started":"2024-02-24T13:33:50.113576Z","shell.execute_reply":"2024-02-24T13:33:50.118646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are going to check the columns from the list above in the train set. There are two options possible\n\n- the respective column has **zero** missing values in the train set; in such a case, we can retain such a column in the subset of parameters for the ML pipelines/experiments down the road;\n- the respective column has **more then `21%`** of missing values in the train set; in such a case, we disqualify this attribute as well as exclude it from the ML experiments down the road. \n\n**Inference:** Based on the assessment of the missing value ratio in the training set, we can conclude the following\n\n- `commnoinclast6m_3546845L` to be disqualified as it has **more then `21%`** of missing values in the train set;\n- `maxdpdfrom6mto36m_3546853P` to be disqualified as it has **more then `21%`** of missing values in the train set.\n\nWith the all of the intelligence collected above, we can make a final call on the subset of the **Static 0** features to be potentially usable in the ML experiments down the road.","metadata":{}},{"cell_type":"markdown","source":"## Static 0 Data: Data Processing and ML implications
\n\nBased on the inspection of the **Static 0 data** as per the sections above, we disqualified a big chunk of attributes due to the data quality issues (major ratio of missing values either in training and/or testing data). We have also identified the subset of good-to-go features that can be used in ML experiments down the road. They are classified in two groups \n\n- attributes without missing values\n- attributes with a reasonably low ratio of missing values that can qualify for ML experiments after the appropriate imputation of the missing values\n\nThe ultimate list of the attributes without missing values to be used in ML experiments is provided below","metadata":{}},{"cell_type":"code","source":"cols_with_full_data = setdiff_sorted(train_cols_with_full_data, full_col_diff, assume_unique=True)\nfor col in cols_with_full_data:\n PrintColor(col)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.121369Z","iopub.execute_input":"2024-02-24T13:33:50.122571Z","iopub.status.idle":"2024-02-24T13:33:50.130743Z","shell.execute_reply.started":"2024-02-24T13:33:50.122525Z","shell.execute_reply":"2024-02-24T13:33:50.129558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The ultimate list of the attributes with a reasonably low ratio of missing values that can qualify for ML experiments after the appropriate imputation of the missing values is provided below","metadata":{}},{"cell_type":"code","source":"disqualified_minor_nan = [\n 'lastapplicationdate_877D',\n 'lastst_736L',\n 'opencred_647L',\n 'paytype1st_925L',\n 'paytype_783L',\n 'price_1097A',\n]\n\ncols_to_impute_data = setdiff_sorted(train_cols_with_minor_nan, disqualified_minor_nan,assume_unique=True)\nfor col in cols_to_impute_data:\n PrintColor(col)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.132172Z","iopub.execute_input":"2024-02-24T13:33:50.133008Z","iopub.status.idle":"2024-02-24T13:33:50.142044Z","shell.execute_reply.started":"2024-02-24T13:33:50.132970Z","shell.execute_reply":"2024-02-24T13:33:50.140021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After the data quality assessment we completed in the sections above, it is the right time to \n- define the data imputation strategy for the columns listed above\n- draw the insightful data visualizations on the **Static 0 data** in the training and visible part of the test sets.","metadata":{}},{"cell_type":"markdown","source":"# Static 0 Data: Data Imputation Strategy
\n\nWe are going to review the summary statistics for each of the variables in the list of the attributes with a reasonably low ratio of missing values in the training set to define the optimal data imputation approach for each of the attributes listed.","metadata":{}},{"cell_type":"markdown","source":"## Imputation of Missing Values for 'annuitynextmonth_57A' attribute
\n \n`annuitynextmonth_57A` stands for *next month's amount of annuity*. It \n- is a numeric variable;\n- has `0.000262%` of missing values in the training dataset\n\nLet's review the essential statistics for this variable to decide on the right imputation strategy.","metadata":{}},{"cell_type":"code","source":"column_name = 'annuitynextmonth_57A'\nfig = build_boxplot(col_name=column_name)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:50.143798Z","iopub.execute_input":"2024-02-24T13:33:50.144445Z","iopub.status.idle":"2024-02-24T13:33:51.983307Z","shell.execute_reply.started":"2024-02-24T13:33:50.144399Z","shell.execute_reply":"2024-02-24T13:33:51.980865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = build_histogram(col_name=column_name)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:51.984802Z","iopub.execute_input":"2024-02-24T13:33:51.985167Z","iopub.status.idle":"2024-02-24T13:33:52.989722Z","shell.execute_reply.started":"2024-02-24T13:33:51.985136Z","shell.execute_reply":"2024-02-24T13:33:52.988564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, imputation with *median value* could be a good strategy for this attribute","metadata":{}},{"cell_type":"code","source":"static_0_df[column_name] = static_0_df[column_name].fillna(static_0_df[column_name].median())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:52.990933Z","iopub.execute_input":"2024-02-24T13:33:52.991267Z","iopub.status.idle":"2024-02-24T13:33:53.028222Z","shell.execute_reply.started":"2024-02-24T13:33:52.991240Z","shell.execute_reply":"2024-02-24T13:33:53.026922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Returns the number of\n# objects it has collected\n# and deallocated\ncollected = gc.collect()\n \n# Prints Garbage collector\n# as 0 object\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)# Importing gc module","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.029797Z","iopub.execute_input":"2024-02-24T13:33:53.030148Z","iopub.status.idle":"2024-02-24T13:33:53.212143Z","shell.execute_reply.started":"2024-02-24T13:33:53.030119Z","shell.execute_reply":"2024-02-24T13:33:53.210799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imputation of Missing Values for 'credtype_322L' attribute
\n\n`credtype_322L` stands for the type of the credit.\n\nIt\n- is of *string* data type\n- has `0.000066%` of missing values\n\nBased on the findings above, the imputation with the most frequent value would be the appropriate strategy.\n\nTBD.","metadata":{}},{"cell_type":"markdown","source":"## Imputation of Missing Values for 'currdebt_22A' attribute
\n\n`currdebt_22A` stands for the current debt amount of the client.\n\nIt\n- is of numeric data type\n- has `0.000262%` of missing values\n\nLet's review the summary statistics for this attribute in the training set.","metadata":{}},{"cell_type":"code","source":"column_name = 'currdebt_22A'\nfig = build_boxplot(col_name=column_name)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.213678Z","iopub.execute_input":"2024-02-24T13:33:53.214619Z","iopub.status.idle":"2024-02-24T13:33:53.360743Z","shell.execute_reply.started":"2024-02-24T13:33:53.214563Z","shell.execute_reply":"2024-02-24T13:33:53.359557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = build_histogram(col_name=column_name)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.362016Z","iopub.execute_input":"2024-02-24T13:33:53.362367Z","iopub.status.idle":"2024-02-24T13:33:53.493681Z","shell.execute_reply.started":"2024-02-24T13:33:53.362338Z","shell.execute_reply":"2024-02-24T13:33:53.492522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, imputation with *median value* could be a good strategy for this attribute","metadata":{}},{"cell_type":"code","source":"static_0_df[column_name] = static_0_df[column_name].fillna(static_0_df[column_name].median())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.495448Z","iopub.execute_input":"2024-02-24T13:33:53.496233Z","iopub.status.idle":"2024-02-24T13:33:53.534208Z","shell.execute_reply.started":"2024-02-24T13:33:53.496186Z","shell.execute_reply":"2024-02-24T13:33:53.532933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Returns the number of\n# objects it has collected\n# and deallocated\ncollected = gc.collect()\n \n# Prints Garbage collector\n# as 0 object\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)# Importing gc module","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.535563Z","iopub.execute_input":"2024-02-24T13:33:53.535905Z","iopub.status.idle":"2024-02-24T13:33:53.711900Z","shell.execute_reply.started":"2024-02-24T13:33:53.535877Z","shell.execute_reply":"2024-02-24T13:33:53.710581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imputation of Missing Values for 'currdebtcredtyperange_828A' attribute
\n\n`currdebtcredtyperange_828A` stands for current amount of debt of the applicant.\n\nIt \n- is of numeric data type\n- has `0.000262%` of missing values\n\nLet's review the summary statistics for this attribute in the training set.","metadata":{}},{"cell_type":"code","source":"column_name = 'currdebtcredtyperange_828A'\nfig = build_boxplot(col_name=column_name)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.714121Z","iopub.execute_input":"2024-02-24T13:33:53.714523Z","iopub.status.idle":"2024-02-24T13:33:53.856848Z","shell.execute_reply.started":"2024-02-24T13:33:53.714457Z","shell.execute_reply":"2024-02-24T13:33:53.855694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = build_histogram(col_name=column_name)\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:53.858608Z","iopub.execute_input":"2024-02-24T13:33:53.859331Z","iopub.status.idle":"2024-02-24T13:33:55.497290Z","shell.execute_reply.started":"2024-02-24T13:33:53.859280Z","shell.execute_reply":"2024-02-24T13:33:55.496217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, imputation with *median value* could be a good strategy for this attribute","metadata":{}},{"cell_type":"code","source":"static_0_df[column_name] = static_0_df[column_name].fillna(static_0_df[column_name].median())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:55.498624Z","iopub.execute_input":"2024-02-24T13:33:55.499573Z","iopub.status.idle":"2024-02-24T13:33:55.531109Z","shell.execute_reply.started":"2024-02-24T13:33:55.499540Z","shell.execute_reply":"2024-02-24T13:33:55.529799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Returns the number of\n# objects it has collected\n# and deallocated\ncollected = gc.collect()\n \n# Prints Garbage collector\n# as 0 object\nprint(\"Garbage collector: collected\",\n \"%d objects.\" % collected)# Importing gc module","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-02-24T13:33:55.532832Z","iopub.execute_input":"2024-02-24T13:33:55.534027Z","iopub.status.idle":"2024-02-24T13:33:55.708951Z","shell.execute_reply.started":"2024-02-24T13:33:55.533983Z","shell.execute_reply":"2024-02-24T13:33:55.707488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imputation of Missing Values for 'disbursementtype_67L' attribute
\n\n`disbursementtype_67L` stands for the type of disbursement.\n\nIt\n- is of *string* data type\n- has `0.056725%` of missing values\n\nBased on the findings above, the imputation with the most frequent value would be the appropriate strategy.\n\nTBD.","metadata":{}},{"cell_type":"markdown","source":"# Static 0 Data: Univariate Analysis of Essential Features (training set)
","metadata":{}},{"cell_type":"markdown","source":"TBD","metadata":{}},{"cell_type":"markdown","source":"# Additional Insights
\n\nTBD in the future versions of the notebook.","metadata":{}}]}