{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nimport os\nimport gc \nfrom IPython.display import display, HTML","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-05T07:56:06.688268Z","iopub.execute_input":"2024-03-05T07:56:06.689141Z","iopub.status.idle":"2024-03-05T07:56:09.512240Z","shell.execute_reply.started":"2024-03-05T07:56:06.689091Z","shell.execute_reply":"2024-03-05T07:56:09.511004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code for read . DRY principle !!","metadata":{}},{"cell_type":"code","source":"directory = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/\"  # Update this with your directory path\nCreditFilesList = os.listdir(directory)\nCreditFilesList.sort()\n\n#for file_name in CreditFilesList:\n    #print(file_name, '\\n\\n')\n    # df = pd.read_csv(os.path.join(directory, file_name))\n    # display(df.head(5))\n\nprint(CreditFilesList)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T07:56:14.496870Z","iopub.execute_input":"2024-03-05T07:56:14.497413Z","iopub.status.idle":"2024-03-05T07:56:14.510647Z","shell.execute_reply.started":"2024-03-05T07:56:14.497385Z","shell.execute_reply":"2024-03-05T07:56:14.509727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Number of columns in a csv ","metadata":{}},{"cell_type":"code","source":"# for files in CreditFilesList:\n#     df = pd.read_csv(os.path.join(directory,files))\n#     print(files)\n#     print(df.columns.nunique())\n#     del df\n#     gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.348196Z","iopub.execute_input":"2024-03-03T13:44:34.349009Z","iopub.status.idle":"2024-03-03T13:44:34.353723Z","shell.execute_reply.started":"2024-03-03T13:44:34.348971Z","shell.execute_reply":"2024-03-03T13:44:34.352564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_applprev_1_0.csv\n41\n\ntrain_applprev_1_1.csv\n41\n\ntrain_applprev_2.csv\n6\n\ntrain_base.csv\n5\n\ntrain_credit_bureau_a_1_0.csv\n79\n\ntrain_credit_bureau_a_1_1.csv\n79\n\ntrain_credit_bureau_a_1_2.csv\n79\n\ntrain_credit_bureau_a_1_3.csv\n79\n\ntrain_credit_bureau_a_1_0.csv\n79\n\ntrain_credit_bureau_a_1_1.csv\n79\n\ntrain_credit_bureau_a_1_2.csv\n79\n\ntrain_credit_bureau_a_1_3.csv\n79\n\ntrain_credit_bureau_a_2_0.csv\n19\n\ntrain_credit_bureau_a_2_1.csv\n19\n\ntrain_credit_bureau_a_2_10.csv\n19\n\ntrain_credit_bureau_a_2_2.csv\n19\n\ntrain_credit_bureau_a_2_3.csv\n19\n\ntrain_credit_bureau_a_2_4.csv\n19\n\ntrain_credit_bureau_a_2_5.csv\n19\n\ntrain_credit_bureau_a_2_6.csv\n19\n\ntrain_credit_bureau_a_2_7.csv\n19\n\ntrain_credit_bureau_a_2_8.csv\n19\n\ntrain_credit_bureau_a_2_9.csv\n19\n\ntrain_credit_bureau_b_1.csv\n45\n\ntrain_credit_bureau_b_2.csv\n6\n\ntrain_debitcard_1.csv\n6\n\ntrain_deposit_1.csv\n5\n\ntrain_other_1.csv\n7\n\ntrain_person_1.csv\n37\n\ntrain_person_2.csv\n11\n\ntrain_static_0_0.csv\n168\n\ntrain_static_0_1.csv\n168\n\ntrain_static_cb_0.csv\n53\n\ntrain_tax_registry_a_1.csv\n5\n\ntrain_tax_registry_b_1.csv\n5\n\ntrain_tax_registry_c_1.csv\n5","metadata":{}},{"cell_type":"markdown","source":"# Let split lists , isnt it ?\n\nNow we can split them in list , as we know number of columns , and from both name and number of columns , we expect that they are similar in a list ","metadata":{}},{"cell_type":"code","source":"applprevlist = ['train_applprev_1_0.csv', 'train_applprev_1_1.csv']\ncredit_bureau_list_1 = [ 'train_credit_bureau_a_2_0.csv', 'train_credit_bureau_a_2_1.csv', 'train_credit_bureau_a_2_10.csv', 'train_credit_bureau_a_2_2.csv', 'train_credit_bureau_a_2_3.csv', 'train_credit_bureau_a_2_4.csv', 'train_credit_bureau_a_2_5.csv', 'train_credit_bureau_a_2_6.csv', 'train_credit_bureau_a_2_7.csv', 'train_credit_bureau_a_2_8.csv', 'train_credit_bureau_a_2_9.csv']\ncredit_bureau_list_2 = ['train_credit_bureau_a_1_0.csv','train_credit_bureau_a_1_1.csv','train_credit_bureau_a_1_2.csv','train_credit_bureau_a_1_3.csv']\nstaticlist  = [ 'train_static_0_0.csv', 'train_static_0_1.csv']\ntax_registList = [ 'train_tax_registry_a_1.csv', 'train_tax_registry_b_1.csv', 'train_tax_registry_c_1.csv']\n# for others we dont need lists ","metadata":{"execution":{"iopub.status.busy":"2024-03-05T07:56:20.891989Z","iopub.execute_input":"2024-03-05T07:56:20.892377Z","iopub.status.idle":"2024-03-05T07:56:20.898935Z","shell.execute_reply.started":"2024-03-05T07:56:20.892347Z","shell.execute_reply":"2024-03-05T07:56:20.897748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I am shocked about inconsistancy in data . ","metadata":{}},{"cell_type":"markdown","source":"# Need to know meanings ","metadata":{}},{"cell_type":"markdown","source":"Meanings baboo meanings ","metadata":{}},{"cell_type":"code","source":"def meaning_mapper(thing_to_map):  \n    df_shorts = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')\n    variable_description_dict = df_shorts.set_index('Variable')['Description'].to_dict()\n    if thing_to_map in variable_description_dict:\n        description = variable_description_dict[thing_to_map]\n        print(f\" == : {thing_to_map} == Description : {description}\")\n        result = f'{description}'\n    else:\n        print(f\"No description found for thing_to_map '{thing_to_map}'\")\n        result = f' {thing_to_map}'\n    return result","metadata":{"execution":{"iopub.status.busy":"2024-03-05T07:56:28.208451Z","iopub.execute_input":"2024-03-05T07:56:28.208865Z","iopub.status.idle":"2024-03-05T07:56:28.215800Z","shell.execute_reply.started":"2024-03-05T07:56:28.208830Z","shell.execute_reply":"2024-03-05T07:56:28.214516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic reporter ","metadata":{}},{"cell_type":"markdown","source":"Why to repeat ? lets make reporter ","metadata":{}},{"cell_type":"code","source":"def file_reporter(df):\n    display(df.head(5))   \n    print(df.shape)\n    print('\\n\\ncolumns\\n\\n')\n    for column in df.columns:\n        meaning_mapper(column)\n    print('\\n\\n Nulls ','\\n\\n')\n    nulls = df.isnull().sum()\n    display(nulls)\n    print('\\n\\n value counts','\\n\\n')\n    vcs = df.value_counts()\n    display(vcs)\n    print('\\n\\n unique no','\\n\\n')\n    unq = df.nunique()\n    display(unq)\n    print('\\n\\n describe\\n\\n')\n    desc = df.describe()\n    display(desc)\n    print('\\n\\n info\\n\\n')\n    info = df.info()\n    display(info)\n    print('\\n\\n\\n\\n')","metadata":{"execution":{"iopub.status.busy":"2024-03-05T07:56:31.407901Z","iopub.execute_input":"2024-03-05T07:56:31.408283Z","iopub.status.idle":"2024-03-05T07:56:31.416084Z","shell.execute_reply.started":"2024-03-05T07:56:31.408252Z","shell.execute_reply":"2024-03-05T07:56:31.414610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mini visuallizer","metadata":{}},{"cell_type":"markdown","source":"Why to write again ? visuallizer of data destribution ","metadata":{}},{"cell_type":"markdown","source":"### Seaborn barplot","metadata":{}},{"cell_type":"code","source":"def Bar_Plotter(\n    df,\n    X_axis,\n    Y_axis,\n    Hue,\n    errorbar=None,\n    capsize=None,\n    palette=None,\n    figsize=None,\n    title=None,\n    X_label=None,\n    Y_label=None,\n    Hue_label=None,\n    grid=True\n):\n    if errorbar is None:\n        errorbar = 'sd'\n    if palette is None:\n        palette = \"Set1\"\n    if capsize is None:\n        capsize = 0.1\n    if figsize is None:\n        figsize = (10, 6)\n    plt.figure(figsize=figsize)\n    sns.barplot(x=df[str(X_axis)], y=df[str(Y_axis)], hue=df[str(\n        Hue)], errorbar=errorbar, capsize=capsize, palette=palette, data=df)\n    if title:\n        plt.title(title)\n    if X_label:\n        plt.xlabel(X_label)\n    if Y_label:\n        plt.ylabel(Y_label)\n    if Hue_label:\n        plt.legend(title=Hue_label)\n    if grid:\n        plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.394813Z","iopub.execute_input":"2024-03-03T13:44:34.395202Z","iopub.status.idle":"2024-03-03T13:44:34.407279Z","shell.execute_reply.started":"2024-03-03T13:44:34.395170Z","shell.execute_reply":"2024-03-03T13:44:34.405664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Seaborn countplot","metadata":{}},{"cell_type":"code","source":"def Count_Plotter(\n    df, \n    X_axis, \n    Hue, \n    palette=None, \n    figsize=None, \n    title=None, \n    X_label=None, \n    Hue_label=None, \n    grid=True, \n    bar_width=0.8\n):\n    try:\n        if palette is None:\n            palette = \"Set1\"\n        if figsize is None:\n            figsize = (10, 6)\n        fig, ax = plt.subplots(figsize=figsize)\n        sns.countplot(x=X_axis, \n                      hue=Hue, \n                      palette=palette, \n                      data=df, \n                      ax=ax)\n        if title:\n            ax.set_title(title)\n        if X_label:\n            ax.set_xlabel(X_label)  \n        if Hue_label:\n            ax.legend(title=Hue_label) \n        if grid:\n            ax.grid()\n        plt.show()\n    except ValueError as e:\n        print(\"Error: Unable to create count plot. Check if DataFrame or columns are empty.\")\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.409326Z","iopub.execute_input":"2024-03-03T13:44:34.409852Z","iopub.status.idle":"2024-03-03T13:44:34.421701Z","shell.execute_reply.started":"2024-03-03T13:44:34.409813Z","shell.execute_reply":"2024-03-03T13:44:34.420026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### seaborn scatterplot","metadata":{}},{"cell_type":"code","source":"def Scatter_Plotter(\n    df,\n    X_axis,\n    Y_axis,\n    Hue,\n    palette=None,\n    figsize=None,\n    title=None,\n    X_label=None,\n    Y_label=None,\n    Hue_label=None,\n    grid=True,\n    on_size=None,\n    range_size=None,\n    leg_label=None,\n    loc=None,\n    bta=None,\n    ncols=None,\n):\n    if palette is None:\n        palette = \"Set1\"\n    if figsize is None:\n        figsize = (10, 6)\n    plt.figure(figsize=figsize) \n    sns.scatterplot(x=X_axis,\n                    y=Y_axis, \n                    hue=Hue, \n                    size=on_size,  \n                    sizes=range_size, \n                    data=df,\n                    label=leg_label)\n    plt.legend(loc=loc, bbox_to_anchor=bta, ncol=ncols)  \n    if title:\n        plt.title(title)\n    if X_label:\n        plt.xlabel(X_label)\n    if Y_label:\n        plt.ylabel(Y_label)\n    if Hue_label:\n        plt.legend(title=Hue_label)\n    if grid:\n        plt.grid()\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.423039Z","iopub.execute_input":"2024-03-03T13:44:34.424159Z","iopub.status.idle":"2024-03-03T13:44:34.438484Z","shell.execute_reply.started":"2024-03-03T13:44:34.424121Z","shell.execute_reply":"2024-03-03T13:44:34.437426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matplotlib parallel bar plot ","metadata":{}},{"cell_type":"code","source":"def parallel_bar_plot_index(\n    dataframe, \n    column1=None, \n    column2=None, \n    num_rows=None, \n    figsize=None,\n    X_label=None,\n    Y_label=None,\n    Title=None,\n    grid=False,\n    color=[],\n):\n    if figsize is None:\n        plt.figure(figsize=(8, 15))\n    else:\n        plt.figure(figsize=figsize)\n    bars1 = plt.barh(dataframe.index, dataframe[column1], label=column1, color=color[0] if color else None)\n    bars2 = plt.barh(dataframe.index, dataframe[column2], label=column2, alpha=0.5, color=color[1] if len(color) > 1 else None)\n    if grid:\n        plt.grid(True)\n    plt.xlabel(X_label)\n    plt.ylabel(Y_label)\n    plt.title(Title)\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.439706Z","iopub.execute_input":"2024-03-03T13:44:34.440510Z","iopub.status.idle":"2024-03-03T13:44:34.451975Z","shell.execute_reply.started":"2024-03-03T13:44:34.440473Z","shell.execute_reply":"2024-03-03T13:44:34.450656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datatypes Converter ","metadata":{}},{"cell_type":"markdown","source":"Convert the datatypes ","metadata":{}},{"cell_type":"code","source":"def datatype_converter(df):\n    for col in df.columns:\n        if df[col].dtype == 'object':\n            try:\n                df[col] = pd.to_datetime(df[col], format='%d-%m-%y')\n            except:\n                try:\n                    df[col] = pd.to_numeric(df[col])\n                except:\n                    df[col] = df[col].astype('category')\n        elif df[col].dtype == 'category':\n            df[col] = df[col].astype('category')\n        else:\n            df[col] = pd.to_numeric(df[col], errors='coerce')  \n    display(df.dtypes)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.453841Z","iopub.execute_input":"2024-03-03T13:44:34.455136Z","iopub.status.idle":"2024-03-03T13:44:34.468091Z","shell.execute_reply.started":"2024-03-03T13:44:34.455086Z","shell.execute_reply":"2024-03-03T13:44:34.466804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlator","metadata":{}},{"cell_type":"code","source":"def correlation_heatmap(df):\n    num_cols = [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]\n    numdf = df[num_cols]\n    plt.figure(figsize=(18,18))\n    sns.heatmap(numdf.corr(), annot=True, fmt=\".1f\")","metadata":{"execution":{"iopub.status.busy":"2024-03-05T08:06:40.594456Z","iopub.execute_input":"2024-03-05T08:06:40.594836Z","iopub.status.idle":"2024-03-05T08:06:40.601193Z","shell.execute_reply.started":"2024-03-05T08:06:40.594807Z","shell.execute_reply":"2024-03-05T08:06:40.599806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Null elemination ","metadata":{}},{"cell_type":"markdown","source":"## Median null eleminator","metadata":{}},{"cell_type":"code","source":"def median_null_eleminator(df):\n    for col in df.columns:\n        if df[col].dtype == 'int64' or df[col].dtype == 'float64':\n            df[col] = df[col].fillna(df[col].median())\n    print('\\n\\nAfter elemination of nulls \\n\\n')\n    display(df.isnull().sum())\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.481672Z","iopub.execute_input":"2024-03-03T13:44:34.482795Z","iopub.status.idle":"2024-03-03T13:44:34.491312Z","shell.execute_reply.started":"2024-03-03T13:44:34.482750Z","shell.execute_reply":"2024-03-03T13:44:34.490079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_rows_percent(df, to_drop):\n    null_props = df.isnull().mean(axis=1)\n    filtered_df = df[null_props < to_drop]\n    return filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.495959Z","iopub.execute_input":"2024-03-03T13:44:34.496322Z","iopub.status.idle":"2024-03-03T13:44:34.502507Z","shell.execute_reply.started":"2024-03-03T13:44:34.496294Z","shell.execute_reply":"2024-03-03T13:44:34.501363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_rows_high_nan(df, threshold):\n    nan_threshold = df.shape[1] * threshold\n    df_filtered = df.dropna(thresh=nan_threshold)\n    return df_filtered","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.504057Z","iopub.execute_input":"2024-03-03T13:44:34.505018Z","iopub.status.idle":"2024-03-03T13:44:34.514917Z","shell.execute_reply.started":"2024-03-03T13:44:34.504984Z","shell.execute_reply":"2024-03-03T13:44:34.513485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main visual handler","metadata":{}},{"cell_type":"markdown","source":"# appleprev list","metadata":{}},{"cell_type":"code","source":"megadf1 = pd.DataFrame()\nfor file in applprevlist:\n    df = pd.read_csv(os.path.join(directory, file))\n    print(file)\n    df = drop_rows_high_nan(df, 0.75)\n    megadf1 = pd.concat([megadf1, df], ignore_index=True) \n    del df \nfile_reporter(megadf1)\ncorrelation_heatmap(megadf1)\nBar_Plotter(df=megadf1.head(1000), \n            X_axis='byoccupationinc_3656910L', \n            Y_axis='childnum_21L', \n            Hue='childnum_21L',\n            figsize=(25,10),\n            title='Applicants income / childrens ', \n            X_label=meaning_mapper('byoccupationinc_3656910L'), \n            Y_label=meaning_mapper('childnum_21L'),\n            Hue_label=meaning_mapper('childnum_21L'))\nCount_Plotter(df=megadf1.head(1000),\n              X_axis='cancelreason_3545846M',\n              Hue='cancelreason_3545846M',\n              palette='Set1',\n              figsize=(25,10),\n              title='Cancellation Reasons',\n              X_label=meaning_mapper('cancelreason_3545846M'),\n              Hue_label=meaning_mapper('cancelreason_3545846M'),\n              bar_width = 1.5,\n             )\nScatter_Plotter(\n    df=megadf1.head(200000),\n    X_axis='credacc_maxhisbal_375A',\n    Y_axis='credacc_minhisbal_90A',\n    Hue='credacc_credlmt_575A',\n    on_size='credacc_actualbalance_314A',\n    range_size=(0, 1500),\n    figsize=(25, 15),\n    loc='upper left',\n    bta=(1, 1),\n    ncols=4,\n    leg_label=meaning_mapper('credacc_credlmt_575A'),\n    X_label = meaning_mapper('credacc_maxhisbal_375A'),\n    Y_label = meaning_mapper('credacc_minhisbal_90A'),\n    )\nBar_Plotter(df=megadf1.head(20000), \n            X_axis='credacc_transactions_402L', \n            Y_axis='credamount_590A', \n            Hue='credacc_transactions_402L',\n            figsize=(25,10),\n            title='number of tranjections / credit amount ', \n            X_label=meaning_mapper('credacc_transactions_402L'), \n            Y_label=meaning_mapper('credamount_590A'),\n            Hue_label=meaning_mapper('credacc_transactions_402L'))\nCount_Plotter(df=megadf1,\n              X_axis='credtype_587L',\n              Hue='credtype_587L',\n              palette='Set1',\n              figsize=(25,5),\n              title='Credit type of  previous application ',\n              X_label=meaning_mapper('credtype_587L'),\n              Hue_label=meaning_mapper('credtype_587L'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf1,\n              X_axis='credacc_status_367L',\n              Hue='credacc_status_367L',\n              palette='Set1',\n              figsize=(25,6),\n              title='dist of Credit status of previous application ',\n              X_label=meaning_mapper('credacc_status_367L'),\n              Hue_label=meaning_mapper('credacc_status_367L'),\n              bar_width = 1.5,\n             )\nBar_Plotter(df=megadf1.head(20000), \n            X_axis='currdebt_94A', \n            Y_axis='credtype_587L', \n            Hue='credtype_587L',\n            figsize=(25,6),\n            title='Credit type and previous application current debt comparision  ', \n            X_label=meaning_mapper('currdebt_94A'), \n            Y_label=meaning_mapper('credtype_587L'),\n            Hue_label=meaning_mapper('credtype_587L'))\nScatter_Plotter(\n    df=megadf1.head(200000),\n    X_axis='byoccupationinc_3656910L',\n    Y_axis='downpmt_134A',\n    Hue='credacc_credlmt_575A',\n    on_size='credacc_credlmt_575A',\n    range_size=(0, 1500),\n    figsize=(25, 5),\n    loc='upper left',\n    bta=(1, 1),\n    ncols=1,\n    leg_label=meaning_mapper('credacc_credlmt_575A'),\n    X_label = meaning_mapper('byoccupationinc_3656910L'),\n    Y_label = meaning_mapper('downpmt_134A'),\n    )\nBar_Plotter(df=megadf1, \n            X_axis='byoccupationinc_3656910L', \n            Y_axis='education_1138M', \n            Hue='education_1138M',\n            figsize=(25,8),\n            title='Applicants education level  ', \n            X_label=meaning_mapper('byoccupationinc_3656910L'), \n            Y_label=meaning_mapper('education_1138M'),\n            Hue_label=meaning_mapper('education_1138M'))\nCount_Plotter(df=megadf1,\n              X_axis='inittransactioncode_279L',\n              Hue='inittransactioncode_279L',\n              palette='Set3',\n              figsize=(25,5),\n              title='Type of initial tranjections made by applicants  ',\n              X_label=meaning_mapper('inittransactioncode_279L'),\n              Hue_label=meaning_mapper('inittransactioncode_279L'),\n              bar_width = 1.5,\n             )\nBar_Plotter(df=megadf1, \n            X_axis='byoccupationinc_3656910L', \n            Y_axis='education_1138M', \n            Hue='education_1138M',\n            figsize=(25,8),\n            title='Applicants education level  ', \n            X_label=meaning_mapper('byoccupationinc_3656910L'), \n            Y_label=meaning_mapper('education_1138M'),\n            Hue_label=meaning_mapper('education_1138M'))\nScatter_Plotter(\n    df=megadf1.head(200000),\n    X_axis='byoccupationinc_3656910L',\n    Y_axis='mainoccupationinc_437A',\n    Hue='mainoccupationinc_437A',\n    on_size='mainoccupationinc_437A',\n    range_size=(0, 1500),\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    leg_label=meaning_mapper('mainoccupationinc_437A'),\n    X_label = meaning_mapper('byoccupationinc_3656910L'),\n    Y_label = meaning_mapper('mainoccupationinc_437A'),\n    )\nCount_Plotter(df=megadf1,\n              X_axis='inittransactioncode_279L',\n              Hue='inittransactioncode_279L',\n              palette='Set3',\n              figsize=(25,5),\n              title='Type of initial tranjections made by applicants  ',\n              X_label=meaning_mapper('inittransactioncode_279L'),\n              Hue_label=meaning_mapper('inittransactioncode_279L'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf1.head(1000),\n              X_axis='profession_152M',\n              Hue='profession_152M',\n              palette='Set3',\n              figsize=(25,5),\n              title='Profession of appplicants  ',\n              X_label=meaning_mapper('profession_152M'),\n              Hue_label=meaning_mapper('profession_152M'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf1.head(1000000),\n              X_axis='rejectreasonclient_4145042M',\n              Hue='rejectreasonclient_4145042M',\n              palette='Set2',\n              figsize=(25,5),\n              title=' Reason for rejection of the clients previous application.  ',\n              X_label=meaning_mapper('rejectreasonclient_4145042M'),\n              Hue_label=meaning_mapper('rejectreasonclient_4145042M'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf1,\n              X_axis='status_219L',\n              Hue='status_219L',\n              palette='Set2',\n              figsize=(25,5),\n              title=' Previous application status.  ',\n              X_label=meaning_mapper('status_219L'),\n              Hue_label=meaning_mapper('status_219L'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf1.head(1000),\n              X_axis='tenor_203L',\n              Hue='tenor_203L',\n              palette='Set1',\n              figsize=(25,5),\n              title='Number of instalments in the previous application.  ',\n              X_label=meaning_mapper('tenor_203L'),\n              Hue_label=meaning_mapper('tenor_203L'),\n              bar_width = 1.5,\n             )\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:44:34.517162Z","iopub.execute_input":"2024-03-03T13:44:34.517561Z","iopub.status.idle":"2024-03-03T13:47:41.096839Z","shell.execute_reply.started":"2024-03-03T13:44:34.517511Z","shell.execute_reply":"2024-03-03T13:47:41.095632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# sample test for the columns in bearou datasets 1","metadata":{}},{"cell_type":"code","source":"# for filename in credit_bureau_list[:1]:\n#     path = os.path.join(directory,filename)\n#     df = pd.read_csv(path)\n#     df","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:47:41.098365Z","iopub.execute_input":"2024-03-03T13:47:41.099407Z","iopub.status.idle":"2024-03-03T13:47:41.104799Z","shell.execute_reply.started":"2024-03-03T13:47:41.099359Z","shell.execute_reply":"2024-03-03T13:47:41.103735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"points noted : Close and active \n\nintrest rates \n\nclassification \n\nstatus \n\nlimit \n\ndate \n\ncatagory \n\namount \n\nmaximum days past \n\nfinancial institution \n\ninstallment amount \n\nintrest rate \n\nmonthly installment payment \n\nfrequency \n\npurpose \n\nresidual amount \n\nsubject role \n\ntotal amount \n\ndebt ","metadata":{}},{"cell_type":"code","source":"megadf2 = pd.DataFrame()\nfor file in credit_bureau_list_2:\n    df = pd.read_csv(os.path.join(directory, file))\n    print(file)\n    df = drop_rows_high_nan(df, 0.75)\n    megadf2 = pd.concat([megadf2, df], ignore_index=True) \n    del df \nfile_reporter(megadf2)      \ncorrelation_heatmap(megadf2)\nScatter_Plotter(\n    df=megadf2,\n    X_axis='annualeffectiverate_199L',\n    Y_axis='annualeffectiverate_63L',\n    figsize=(25, 5),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('annualeffectiverate_199L'),\n    Y_label = meaning_mapper('annualeffectiverate_63L'),\n    )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='annualeffectiverate_199L',\n    Y_axis='annualeffectiverate_63L',\n    figsize=(25, 5),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('annualeffectiverate_199L'),\n    Y_label = meaning_mapper('annualeffectiverate_63L'),\n    )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='classificationofcontr_13M',\n    Y_axis='classificationofcontr_400M',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('classificationofcontr_13M'),\n    Y_label = meaning_mapper('classificationofcontr_400M'),\n    grid = False,\n    )\nCount_Plotter(df=megadf2,\n              X_axis='contractst_545M',\n              Hue='contractst_545M',\n              palette='Set3',\n              figsize=(25,5),\n              title=' Contract status.  ',\n              X_label=meaning_mapper('contractst_545M'),\n              Hue_label=meaning_mapper('contractst_545M'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf2.head(200),\n              X_axis='contractst_964M',\n              Hue='contractst_964M',\n              palette='Set3',\n              figsize=(25,5),\n              title='Contract status of terminated credit contract.  ',\n              X_label=meaning_mapper('contractst_964M'),\n              Hue_label=meaning_mapper('contractst_964M'),\n              bar_width = 1.5,\n             )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='credlmt_230A',\n    Y_axis='credlmt_935A',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('credlmt_230A'),\n    Y_label = meaning_mapper('credlmt_935A'),\n    grid = False,\n    )\nCount_Plotter(df=megadf2.head(20),\n              X_axis='debtoutstand_525A',\n              Hue='debtoutstand_525A',\n              palette='Set3',\n              figsize=(25,5),\n              title='Outstanding amount of existing contract.  ',\n              X_label=meaning_mapper('debtoutstand_525A'),\n              Hue_label=meaning_mapper('debtoutstand_525A'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf2.head(100),\n              X_axis='debtoverdue_47A',\n              Hue='debtoverdue_47A',\n              palette='Set1',\n              figsize=(25,5),\n              X_label=meaning_mapper('debtoverdue_47A'),\n              Hue_label=meaning_mapper('debtoverdue_47A'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf2.head(4000),\n              X_axis='description_351M',\n              Hue='description_351M',\n              palette='Set3',\n              figsize=(25,5),\n              title='Profession of appplicants  ',\n              X_label=meaning_mapper('description_351M'),\n              Hue_label=meaning_mapper('description_351M'),\n              bar_width = 1.5,\n             )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='dpdmax_139P',\n    Y_axis='dpdmax_757P',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('dpdmax_139P'),\n    Y_label = meaning_mapper('dpdmax_757P'),\n    grid = False,\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='dpdmaxdatemonth_442T',\n    column2='dpdmaxdatemonth_89T',\n    figsize=(25, 15),\n    X_label=meaning_mapper('dpdmaxdatemonth_442T'),\n    Y_label=meaning_mapper('dpdmaxdatemonth_89T'),\n    Title='Terminated and active contract DPD month',\n    grid=True,\n    color=['blue', 'm']\n    )\nCount_Plotter(df=megadf2.head(500),\n              X_axis='financialinstitution_382M',\n              Hue='financialinstitution_591M',\n              palette='Set3',\n              figsize=(25,5),\n              title='Name of financial institution that is linked to a closed contract, and how they again acquired by the active contract ',\n              X_label=meaning_mapper('financialinstitution_382M'),\n              Hue_label=meaning_mapper('financialinstitution_591M'),\n              bar_width = 1.5,\n             )\n    \nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='instlamount_768A',\n    column2='instlamount_852A',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('instlamount_768A'),\n    Y_label=meaning_mapper('instlamount_852A'),\n    Title='installment amount for active and cloed contract',\n    grid=True,\n    color=['blue', 'orange']\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='nominalrate_281L',\n    column2='nominalrate_498L',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('nominalrate_281L'),\n    Y_label=meaning_mapper('nominalrate_498L'),\n    Title='intrest rate of active and closed contract ',\n    grid=True,\n    color=['blue', 'orange']\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='numberofcontrsvalue_258L',\n    column2='numberofcontrsvalue_358L',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('numberofcontrsvalue_258L'),\n    Y_label=meaning_mapper('numberofcontrsvalue_358L'),\n    Title='Number of active and closed contracts',\n    grid=True,\n    color=['blue', 'orange']\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='numberofinstls_229L',\n    column2='numberofinstls_320L',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('numberofinstls_229L'),\n    Y_label=meaning_mapper('numberofinstls_320L'),\n    Title='number of installments on closed and active contracts',\n    grid=True,\n    color=['blue', 'orange']\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='numberofoutstandinstls_520L',\n    column2='numberofoutstandinstls_59L',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('numberofoutstandinstls_520L'),\n    Y_label=meaning_mapper('numberofoutstandinstls_59L'),\n    Title='outstanding installments of active and closed ocntracts',\n    grid=True,\n    color=['blue', 'm']\n    )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='numberofoverdueinstls_725L',\n    Y_axis='numberofoverdueinstls_834L',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('numberofoverdueinstls_725L'),\n    Y_label = meaning_mapper('numberofoverdueinstls_834L'),\n    grid = False,\n    )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='outstandingamount_354A',\n    Y_axis='outstandingamount_362A',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('outstandingamount_354A'),\n    Y_label = meaning_mapper('outstandingamount_362A'),\n    grid = False,\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='overdueamountmax2_14A',\n    column2='overdueamountmax2_398A',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('overdueamountmax2_14A'),\n    Y_label=meaning_mapper('overdueamountmax2_398A'),\n    Title='Amounts on post dues ',\n    grid=True,\n    color=['blue', 'orange']\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='overdueamountmax_155A',\n    column2='overdueamountmax_35A',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('overdueamountmax_155A'),\n    Y_label=meaning_mapper('overdueamountmax_35A'),\n    Title='Parallel Bar Plot',\n    grid=True,\n    color=['blue', 'orange']\n    )\nparallel_bar_plot_index(\n    dataframe=megadf2.head(500),\n    column1='periodicityofpmts_1102L',\n    column2='periodicityofpmts_837L',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('periodicityofpmts_1102L'),\n    Y_label=meaning_mapper('periodicityofpmts_837L'),\n    Title='frequency of installments for active and closed contracts',\n    grid=True,\n    color=['blue', 'orange']\n    )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='prolongationcount_1120L',\n    Y_axis='prolongationcount_599L',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('prolongationcount_1120L'),\n    Y_label = meaning_mapper('prolongationcount_599L'),\n    grid = False,\n    )\nCount_Plotter(df=megadf2.head(100),\n              X_axis='purposeofcred_426M',\n              Hue='purposeofcred_426M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Purpose of credit for active contract. ',\n              X_label=meaning_mapper('purposeofcred_426M'),\n              Hue_label=meaning_mapper('purposeofcred_426M'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf2.head(100),\n              X_axis='purposeofcred_874M',\n              Hue='purposeofcred_874M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Purpose of credit on a closed contract. ',\n              X_label=meaning_mapper('purposeofcred_874M'),\n              Hue_label=meaning_mapper('purposeofcred_874M'),\n              bar_width = 1.5,\n             )\nScatter_Plotter(\n    df=megadf2,\n    X_axis='residualamount_488A',\n    Y_axis='residualamount_856A',\n    figsize=(25, 15),\n    loc='upper left',\n    ncols=1,\n    Hue = None,\n    X_label = meaning_mapper('residualamount_488A'),\n    Y_label = meaning_mapper('residualamount_856A'),\n    grid = False,\n    )\nCount_Plotter(df=megadf2.head(100),\n              X_axis='subjectrole_182M',\n              Hue='subjectrole_182M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Subject role in active credit contract. ',\n              X_label=meaning_mapper('subjectrole_182M'),\n              Hue_label=meaning_mapper('subjectrole_182M'),\n              bar_width = 1.5,\n             )\nCount_Plotter(df=megadf2.head(100),\n              X_axis='subjectrole_93M',\n              Hue='subjectrole_93M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Purpose of credit on a closed contract.  ',\n              X_label=meaning_mapper('purposeofcred_874M'),\n              Hue_label=meaning_mapper('purposeofcred_874M'),\n              bar_width = 1.5,\n             )\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:47:41.106687Z","iopub.execute_input":"2024-03-03T13:47:41.107096Z","iopub.status.idle":"2024-03-03T13:53:25.503327Z","shell.execute_reply.started":"2024-03-03T13:47:41.107062Z","shell.execute_reply":"2024-03-03T13:53:25.501623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"megadf3 = pd.DataFrame()\nfor file in credit_bureau_list_1:\n    df = pd.read_csv(os.path.join(directory, file))\n    print(file)\n    df = drop_rows_high_nan(df, 0.75)\n    megadf3 = pd.concat([megadf3, df], ignore_index=True) \n    del df \n\nfile_reporter(megadf3)\ncorrelation_heatmap(megadf3)\n    \nparallel_bar_plot_index(\n    dataframe=megadf3.head(5000),\n    column1='collater_typofvalofguarant_298M',\n    column2='collater_typofvalofguarant_407M',\n    num_rows=30000,\n    figsize=(25, 15),\n    X_label=meaning_mapper('collater_typofvalofguarant_298M'),\n    Y_label=meaning_mapper('collater_typofvalofguarant_407M'),\n    Title='Collatral valuation type : Active v/s closed',\n    grid=True,\n    color=['blue']\n    )\n    \nparallel_bar_plot_index(\n    dataframe=megadf3.head(1000),\n    column1='collater_valueofguarantee_1124L',\n    column2='collater_valueofguarantee_876L',\n    figsize=(25, 15),\n    X_label=meaning_mapper('collater_valueofguarantee_1124L'),\n    Y_label=meaning_mapper('collater_valueofguarantee_876L'),\n    Title='valure of collatrals of active and closed ocntracts',\n    grid=True,\n    color=['blue', 'm']\n    )\n    \nCount_Plotter(df=megadf3,\n              X_axis='collaterals_typeofguarante_359M',\n              Hue='collaterals_typeofguarante_359M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Type of collateral that was used as a guarantee for a closed contract.',\n              X_label=meaning_mapper('collaterals_typeofguarante_359M'),\n              Hue_label=meaning_mapper('collaterals_typeofguarante_359M'),\n              bar_width = 1.5,\n             )\n    \nCount_Plotter(df=megadf3,\n              X_axis='collaterals_typeofguarante_669M',\n              Hue='collaterals_typeofguarante_669M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Type of collateral that was used as a guarantee for a active contract.',\n              X_label=meaning_mapper('collaterals_typeofguarante_669M'),\n              Hue_label=meaning_mapper('collaterals_typeofguarante_669M'),\n              bar_width = 1.5,\n             )\n    \nCount_Plotter(df=megadf3,\n              X_axis='subjectroles_name_541M',\n              Hue='subjectroles_name_541M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Name of subject role in closed credit contract (num_group1 - terminated contract, num_group2 - subject roles).',\n              X_label=meaning_mapper('subjectroles_name_541M'),\n              Hue_label=meaning_mapper('subjectroles_name_541M'),\n              bar_width = 1.5,\n             )\n    \nCount_Plotter(df=megadf3,\n              X_axis='subjectroles_name_838M',\n              Hue='subjectroles_name_838M',\n              palette='Set1',\n              figsize=(25,5),\n              title='Name of subject role in active credit contract (num_group1 - existing contract, num_group2 - subject roles). ',\n              X_label=meaning_mapper('subjectroles_name_838M'),\n              Hue_label=meaning_mapper('subjectroles_name_838M'),\n              bar_width = 1.5,\n             )\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-03T13:53:25.505404Z","iopub.execute_input":"2024-03-03T13:53:25.505811Z","iopub.status.idle":"2024-03-03T14:17:03.723747Z","shell.execute_reply.started":"2024-03-03T13:53:25.505774Z","shell.execute_reply":"2024-03-03T14:17:03.722125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"megadf4 = pd.DataFrame()\nfor file in staticlist:\n    path=os.path.join(directory,file)\n    print(path)\n    df = pd.read_csv(path)\n    megadf4 = pd.concat([megadf4, df], ignore_index=True) \n    del df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-05T08:03:48.934007Z","iopub.execute_input":"2024-03-05T08:03:48.934488Z","iopub.status.idle":"2024-03-05T08:04:22.661648Z","shell.execute_reply.started":"2024-03-05T08:03:48.934451Z","shell.execute_reply":"2024-03-05T08:04:22.660645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"megadf4.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T08:04:46.413760Z","iopub.execute_input":"2024-03-05T08:04:46.414210Z","iopub.status.idle":"2024-03-05T08:04:46.444736Z","shell.execute_reply.started":"2024-03-05T08:04:46.414175Z","shell.execute_reply":"2024-03-05T08:04:46.442916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_reporter(megadf4)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T08:04:52.653910Z","iopub.execute_input":"2024-03-05T08:04:52.654368Z","iopub.status.idle":"2024-03-05T08:05:24.451959Z","shell.execute_reply.started":"2024-03-05T08:04:52.654335Z","shell.execute_reply":"2024-03-05T08:05:24.450482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correlation_heatmap(megadf4)","metadata":{"execution":{"iopub.status.busy":"2024-03-05T08:10:57.530563Z","iopub.execute_input":"2024-03-05T08:10:57.532034Z","iopub.status.idle":"2024-03-05T08:10:57.538167Z","shell.execute_reply.started":"2024-03-05T08:10:57.531980Z","shell.execute_reply":"2024-03-05T08:10:57.536316Z"},"trusted":true},"execution_count":null,"outputs":[]}]}