{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is to study any relationship between features and defaults\n\nAs the file size and number of features are large, I took random samples to visualize the relationship. \n\nFurthermore, upon sampling, I did not fix it using a randomstate but instead made it free to see if any changes occur upon the next operation.","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport pyarrow.parquet as pq\nimport pyarrow as pa\nimport gc\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:25:10.005236Z","iopub.execute_input":"2024-02-12T12:25:10.005654Z","iopub.status.idle":"2024-02-12T12:25:10.010451Z","shell.execute_reply.started":"2024-02-12T12:25:10.005628Z","shell.execute_reply":"2024-02-12T12:25:10.009283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet').to_pandas()\nprint(base.columns)\nfeatures = ['case_id', 'target']\nbase_data = base[features]\nprint(base_data.shape)\nprint(base_data.columns)","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:26:27.407717Z","iopub.execute_input":"2024-02-12T12:26:27.408054Z","iopub.status.idle":"2024-02-12T12:26:27.525622Z","shell.execute_reply.started":"2024-02-12T12:26:27.408028Z","shell.execute_reply":"2024-02-12T12:26:27.524936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the Parquet files\ndata1 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_1_0.parquet').to_pandas()\nprint(\"Shape of data1:\", data1.shape)\n\ndata2 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_1_1.parquet').to_pandas()\nprint(\"Shape of data2:\", data2.shape)\ncombined_data = pd.concat([data1, data2], ignore_index=True)\ncombined_data.reset_index(drop=True, inplace=True)\nprint(combined_data.shape)\n\ncombined_data = pd.merge(combined_data, base_data, on='case_id')\nprint(combined_data.shape)\nprint(combined_data.columns)","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:27:59.206970Z","iopub.execute_input":"2024-02-12T12:27:59.207350Z","iopub.status.idle":"2024-02-12T12:28:11.418182Z","shell.execute_reply.started":"2024-02-12T12:27:59.207320Z","shell.execute_reply":"2024-02-12T12:28:11.417247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_count = (combined_data['target'] == 0).sum()\nnon_zero_count = (combined_data['target'] != 0).sum()\n\nprint(\"# of default cases :\", non_zero_count)\nprint(\"# of performing cases :\", zero_count)\n\n# Take a random sample of the data to save memory. Furthermore, do not use a random state as it is designed to change over time\nsampled_data = combined_data.sample(frac=0.002)  \n\ncolumns_to_plot = ['actualdpd_943P', 'annuity_853A', 'approvaldate_319D',\n       'byoccupationinc_3656910L', 'cancelreason_3545846M', 'childnum_21L',\n       'creationdate_885D', 'credacc_actualbalance_314A',\n       'credacc_credlmt_575A', 'credacc_maxhisbal_375A',\n       'credacc_minhisbal_90A', 'credacc_status_367L',\n       'credacc_transactions_402L', 'credamount_590A', 'credtype_587L',\n       'currdebt_94A', ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=sampled_data, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:31:14.719332Z","iopub.execute_input":"2024-02-12T12:31:14.719657Z","iopub.status.idle":"2024-02-12T12:31:51.734068Z","shell.execute_reply.started":"2024-02-12T12:31:14.719630Z","shell.execute_reply":"2024-02-12T12:31:51.732858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ 'dateactivated_425D', 'district_544M', 'downpmt_134A',\n       'dtlastpmt_581D', 'dtlastpmtallstes_3545839D', 'education_1138M',\n       'employedfrom_700D', 'familystate_726L', 'firstnonzeroinstldate_307D',\n       'inittransactioncode_279L', 'isbidproduct_390L', 'isdebitcard_527L',\n       'mainoccupationinc_437A', 'maxdpdtolerance_577P', 'num_group1',\n       'outstandingdebt_522A',  ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=sampled_data, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:34:25.680000Z","iopub.execute_input":"2024-02-12T12:34:25.680330Z","iopub.status.idle":"2024-02-12T12:35:17.594156Z","shell.execute_reply.started":"2024-02-12T12:34:25.680305Z","shell.execute_reply":"2024-02-12T12:35:17.593183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = ['pmtnum_8L', 'postype_4733339M',\n       'profession_152M', 'rejectreason_755M', 'rejectreasonclient_4145042M',\n       'revolvingaccount_394A', 'status_219L', 'tenor_203L']\n\nfig, axes = plt.subplots(nrows=2, ncols=4, figsize=(16, 8))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=sampled_data, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        \n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n    \nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:36:22.631975Z","iopub.execute_input":"2024-02-12T12:36:22.632334Z","iopub.status.idle":"2024-02-12T12:36:26.325885Z","shell.execute_reply.started":"2024-02-12T12:36:22.632307Z","shell.execute_reply":"2024-02-12T12:36:26.325291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data3 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_applprev_2.parquet')\nprint(data3.shape)\nprint(base_data.shape)\n# Merge the dataframes on 'case_id'\ndata3_df = data3.to_pandas()\ndata3_df.dropna(inplace=True)\nprint(data3_df.isnull().sum())\napplprev_2_df = pd.merge(data3_df, base_data, on='case_id')\nprint(applprev_2_df.shape)\nprint(applprev_2_df.columns)\n\n# applprev_2_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:36:49.808988Z","iopub.execute_input":"2024-02-12T12:36:49.810042Z","iopub.status.idle":"2024-02-12T12:36:53.183604Z","shell.execute_reply.started":"2024-02-12T12:36:49.810005Z","shell.execute_reply":"2024-02-12T12:36:53.182745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"applprev_2_df_forplot = applprev_2_df.sample(frac=0.2)  # Adjust the fraction as needed\n\ncolumns_to_plot = ['cacccardblochreas_147M', 'conts_type_509L',\n           'credacc_cards_status_52L', 'num_group1', 'num_group2',]\n\nfig, axes = plt.subplots(nrows=2, ncols=3, figsize=(16, 16))\nplt.xticks(rotation=30)\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=applprev_2_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:37:27.616209Z","iopub.execute_input":"2024-02-12T12:37:27.616509Z","iopub.status.idle":"2024-02-12T12:37:33.795812Z","shell.execute_reply.started":"2024-02-12T12:37:27.616488Z","shell.execute_reply":"2024-02-12T12:37:33.794967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data4 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_cb_0.parquet')\nprint(data4.shape)\n# Merge the dataframes on 'case_id'\ndata4_df = data4.to_pandas()\n\ntrain_static_cb_0_df = pd.merge(data4_df, base_data, on='case_id')\n\nprint(train_static_cb_0_df.shape)\ntrain_static_cb_0_df.dropna(subset=['assignmentdate_238D'], inplace=True)\nprint(train_static_cb_0_df.shape)\nprint(train_static_cb_0_df.columns)\n\ntrain_static_cb_0_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:37:51.528618Z","iopub.execute_input":"2024-02-12T12:37:51.528942Z","iopub.status.idle":"2024-02-12T12:37:53.226356Z","shell.execute_reply.started":"2024-02-12T12:37:51.528917Z","shell.execute_reply":"2024-02-12T12:37:53.225319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_cb_0_forplot = train_static_cb_0_df.sample(frac=0.15)  \n\ncolumns_to_plot = ['assignmentdate_238D', 'assignmentdate_4527235D', 'assignmentdate_4955616D', 'birthdate_574D', \n                   'contractssum_5085716L', 'dateofbirth_337D', 'dateofbirth_342D', 'days120_123L', \n                   'days180_256L', 'days30_165L', 'days360_512L', 'days90_310L', \n                   'description_5085714M', 'education_1103M', 'education_88M', 'firstquarter_103L', ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_cb_0_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:39:53.226773Z","iopub.execute_input":"2024-02-12T12:39:53.227086Z","iopub.status.idle":"2024-02-12T12:40:46.788057Z","shell.execute_reply.started":"2024-02-12T12:39:53.227062Z","shell.execute_reply":"2024-02-12T12:40:46.787078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [\n                   'for3years_128L', 'for3years_504L', 'for3years_584L', 'formonth_118L', \n                   'formonth_206L', 'formonth_535L', 'forquarter_1017L', 'forquarter_462L', \n                   'forquarter_634L', 'fortoday_1092L', 'forweek_1077L', 'forweek_528L', \n                   'forweek_601L', 'foryear_618L', 'foryear_818L', 'foryear_850L', \n                   ]\n\n\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_cb_0_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:41:00.924293Z","iopub.execute_input":"2024-02-12T12:41:00.924629Z","iopub.status.idle":"2024-02-12T12:41:05.355086Z","shell.execute_reply.started":"2024-02-12T12:41:00.924597Z","shell.execute_reply":"2024-02-12T12:41:05.354490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [\n                   'fourthquarter_440L', 'maritalst_385M', 'maritalst_893M', 'numberofqueries_373L', \n                   'pmtaverage_3A', 'pmtaverage_4527227A', 'pmtaverage_4955615A', 'pmtcount_4527229L', \n                   'pmtcount_4955617L', 'pmtcount_693L', 'pmtscount_423L', 'pmtssum_45A', \n                   'requesttype_4525192L', 'responsedate_1012D', 'responsedate_4527233D', 'responsedate_4917613D', \n                   ]\n\n\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_cb_0_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:41:14.743403Z","iopub.execute_input":"2024-02-12T12:41:14.743730Z","iopub.status.idle":"2024-02-12T12:41:21.785204Z","shell.execute_reply.started":"2024-02-12T12:41:14.743704Z","shell.execute_reply":"2024-02-12T12:41:21.784272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ \n                   'riskassesment_302T', 'riskassesment_940T', 'secondquarter_766L', 'thirdquarter_1082L']\n\n\n\nfig, axes = plt.subplots(nrows=2, ncols=2, figsize=(8, 8))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_cb_0_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:41:54.988333Z","iopub.execute_input":"2024-02-12T12:41:54.988681Z","iopub.status.idle":"2024-02-12T12:41:56.597334Z","shell.execute_reply.started":"2024-02-12T12:41:54.988653Z","shell.execute_reply":"2024-02-12T12:41:56.596482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data5 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_2.parquet')\nprint(data5.shape)\n# Merge the dataframes on 'case_id'\ndata5_df = data5.to_pandas()\ntrain_person_2_df = pd.merge(data5_df, base_data, on='case_id')\n\nprint(train_person_2_df.shape)\ntrain_person_2_df.dropna(subset=['addres_district_368M'], inplace=True)\nprint(train_person_2_df.shape)\nprint(train_person_2_df.columns)\n\ntrain_person_2_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:42:03.825023Z","iopub.execute_input":"2024-02-12T12:42:03.825365Z","iopub.status.idle":"2024-02-12T12:42:05.462512Z","shell.execute_reply.started":"2024-02-12T12:42:03.825340Z","shell.execute_reply":"2024-02-12T12:42:05.461249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_2_df_forplot = train_person_2_df.sample(frac=0.02)  \n\ncolumns_to_plot = ['addres_district_368M', 'addres_role_871L', 'addres_zip_823M', \n                   'conts_role_79M', 'empls_economicalst_849M', 'empls_employedfrom_796D', \n                   'empls_employer_name_740M', 'num_group1', 'num_group2', 'relatedpersons_role_762T']\n\nfig, axes = plt.subplots(nrows=4, ncols=3, figsize=(24, 24))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_person_2_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:43:00.254243Z","iopub.execute_input":"2024-02-12T12:43:00.254567Z","iopub.status.idle":"2024-02-12T12:43:10.755761Z","shell.execute_reply.started":"2024-02-12T12:43:00.254542Z","shell.execute_reply":"2024-02-12T12:43:10.754876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data6 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_1.parquet')\nprint(data6.shape)\n# Merge the dataframes on 'case_id'\ndata6_df = data6.to_pandas()\ntrain_credit_bureau_b_1_df = pd.merge(data6_df, base_data, on='case_id')\n\nprint(train_credit_bureau_b_1_df.shape)\ntrain_credit_bureau_b_1_df.dropna(subset=['amount_1115A'], inplace=True)\nprint(train_credit_bureau_b_1_df.shape)\nprint(train_credit_bureau_b_1_df.columns)\n\ntrain_credit_bureau_b_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:43:29.115311Z","iopub.execute_input":"2024-02-12T12:43:29.115627Z","iopub.status.idle":"2024-02-12T12:43:29.491005Z","shell.execute_reply.started":"2024-02-12T12:43:29.115603Z","shell.execute_reply":"2024-02-12T12:43:29.490168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_b_1_df_forplot = train_credit_bureau_b_1_df.sample(frac=0.05)  \n\ncolumns_to_plot = ['amount_1115A', 'classificationofcontr_1114M', 'contractdate_551D', 'contractmaturitydate_151D', \n                   'contractst_516M', 'contracttype_653M', 'credlmt_1052A', 'credlmt_228A', \n                   'credlmt_3940954A', 'credor_3940957M', 'credquantity_1099L', 'credquantity_984L', \n                   'debtpastduevalue_732A', 'debtvalue_227A', 'dpd_550P', 'dpd_733P', \n                   ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_b_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:43:39.987298Z","iopub.execute_input":"2024-02-12T12:43:39.987614Z","iopub.status.idle":"2024-02-12T12:43:54.743223Z","shell.execute_reply.started":"2024-02-12T12:43:39.987588Z","shell.execute_reply":"2024-02-12T12:43:54.742567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [\n                   'installmentamount_833A', 'instlamount_892A', 'interesteffectiverate_369L', 'interestrateyearly_538L', \n                   'lastupdate_260D', 'maxdebtpduevalodued_3940955A', 'num_group1', 'numberofinstls_810L', \n                   'overdueamountmax_950A', 'overdueamountmaxdatemonth_494T', 'overdueamountmaxdateyear_432T', 'periodicityofpmts_997L', \n                   'periodicityofpmts_997M', 'pmtdaysoverdue_1135P', 'pmtmethod_731M', 'pmtnumpending_403L', \n                   ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(24, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_b_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:44:10.726733Z","iopub.execute_input":"2024-02-12T12:44:10.727045Z","iopub.status.idle":"2024-02-12T12:44:16.941101Z","shell.execute_reply.started":"2024-02-12T12:44:10.727022Z","shell.execute_reply":"2024-02-12T12:44:16.939872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [\n                   'purposeofcred_722M', 'residualamount_1093A', 'residualamount_127A', 'residualamount_3940956A', \n                   'subjectrole_326M', 'subjectrole_43M', 'totalamount_503A', 'totalamount_881A'\n                   ]\n\nfig, axes = plt.subplots(nrows=2, ncols=4, figsize=(16, 8))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_b_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:45:05.384432Z","iopub.execute_input":"2024-02-12T12:45:05.384758Z","iopub.status.idle":"2024-02-12T12:45:07.400695Z","shell.execute_reply.started":"2024-02-12T12:45:05.384734Z","shell.execute_reply":"2024-02-12T12:45:07.399888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data7 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_b_2.parquet')\nprint(data7.shape)\n# Merge the dataframes on 'case_id'\ndata7_df = data7.to_pandas()\ntrain_credit_bureau_b_2_df = pd.merge(data7_df, base_data, on='case_id')\n\nprint(train_credit_bureau_b_2_df.shape)\ntrain_credit_bureau_b_2_df.dropna(subset=['num_group1'], inplace=True)\nprint(train_credit_bureau_b_2_df.shape)\nprint(train_credit_bureau_b_2_df.columns)\n\ntrain_credit_bureau_b_2_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:45:40.314130Z","iopub.execute_input":"2024-02-12T12:45:40.314459Z","iopub.status.idle":"2024-02-12T12:45:40.790189Z","shell.execute_reply.started":"2024-02-12T12:45:40.314433Z","shell.execute_reply":"2024-02-12T12:45:40.789158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_b_2_df_forplot = train_credit_bureau_b_2_df.sample(frac=0.05)  \n\ncolumns_to_plot = ['num_group1', 'num_group2', 'pmts_date_1107D', 'pmts_dpdvalue_108P', 'pmts_pmtsoverdue_635A']\n\nfig, axes = plt.subplots(nrows=3, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        plt.xticks(rotation=30)\n        sns.scatterplot(data=train_credit_bureau_b_2_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:46:09.564504Z","iopub.execute_input":"2024-02-12T12:46:09.564823Z","iopub.status.idle":"2024-02-12T12:46:16.013895Z","shell.execute_reply.started":"2024-02-12T12:46:09.564799Z","shell.execute_reply":"2024-02-12T12:46:16.013291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata8 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_1_0.parquet')\nprint(data8.shape)\n# Merge the dataframes on 'case_id'\ndata8_df = data8.to_pandas()\ndata8_df = data8_df.sample(frac=0.2) #for memory control\ntrain_credit_bureau_a_1_0_df = pd.merge(data8_df, base_data, on='case_id')\n\nprint(train_credit_bureau_a_1_0_df.shape)\ntrain_credit_bureau_a_1_0_df.dropna(subset=['annualeffectiverate_199L'], inplace=True)\nprint(train_credit_bureau_a_1_0_df.shape)\nprint(train_credit_bureau_a_1_0_df.columns)\n\ntrain_credit_bureau_a_1_0_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:46:26.006868Z","iopub.execute_input":"2024-02-12T12:46:26.007262Z","iopub.status.idle":"2024-02-12T12:46:33.575772Z","shell.execute_reply.started":"2024-02-12T12:46:26.007229Z","shell.execute_reply":"2024-02-12T12:46:33.574548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_1_0_df_forplot = train_credit_bureau_a_1_0_df.sample(frac=1)  \n\ncolumns_to_plot = [ 'annualeffectiverate_199L', 'annualeffectiverate_63L', 'classificationofcontr_13M', 'classificationofcontr_400M', \n                   'contractst_545M', 'contractst_964M', 'contractsum_5085717L', 'credlmt_230A', \n                   'credlmt_935A', 'dateofcredend_289D', 'dateofcredend_353D', 'dateofcredstart_181D', \n                   'dateofcredstart_739D', 'dateofrealrepmt_138D', 'debtoutstand_525A', 'debtoverdue_47A', ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(24, 20))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_a_1_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:46:49.164077Z","iopub.execute_input":"2024-02-12T12:46:49.164431Z","iopub.status.idle":"2024-02-12T12:47:28.188289Z","shell.execute_reply.started":"2024-02-12T12:46:49.164407Z","shell.execute_reply":"2024-02-12T12:47:28.187333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ \n                   'description_351M', 'dpdmax_139P', 'dpdmax_757P', 'dpdmaxdatemonth_442T', \n                   'dpdmaxdatemonth_89T', 'dpdmaxdateyear_596T', 'dpdmaxdateyear_896T', 'financialinstitution_382M', \n                   'financialinstitution_591M', 'instlamount_768A', 'instlamount_852A', 'interestrate_508L', \n                   'lastupdate_1112D', 'lastupdate_388D', 'monthlyinstlamount_332A', 'monthlyinstlamount_674A', ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(24, 20))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_a_1_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:47:37.480245Z","iopub.execute_input":"2024-02-12T12:47:37.480586Z","iopub.status.idle":"2024-02-12T12:47:49.560853Z","shell.execute_reply.started":"2024-02-12T12:47:37.480557Z","shell.execute_reply":"2024-02-12T12:47:49.559773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ \n                   'nominalrate_281L', 'nominalrate_498L', 'num_group1', 'numberofcontrsvalue_258L', \n                   'numberofcontrsvalue_358L', 'numberofinstls_229L', 'numberofinstls_320L', 'numberofoutstandinstls_520L',\n                   'numberofoutstandinstls_59L', 'numberofoverdueinstlmax_1039L', 'numberofoverdueinstlmax_1151L', 'numberofoverdueinstlmaxdat_148D', \n                   'numberofoverdueinstlmaxdat_641D', 'numberofoverdueinstls_725L', 'numberofoverdueinstls_834L', 'outstandingamount_354A', \n                   ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(24, 20))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_a_1_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:48:09.686060Z","iopub.execute_input":"2024-02-12T12:48:09.686388Z","iopub.status.idle":"2024-02-12T12:48:18.882670Z","shell.execute_reply.started":"2024-02-12T12:48:09.686363Z","shell.execute_reply":"2024-02-12T12:48:18.882034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [\n                   'outstandingamount_362A', 'overdueamount_31A', 'overdueamount_659A', 'overdueamountmax2_14A', \n                   'overdueamountmax2_398A', 'overdueamountmax2date_1002D', 'overdueamountmax2date_1142D', 'overdueamountmax_155A', \n                   'overdueamountmax_35A', 'overdueamountmaxdatemonth_284T', 'overdueamountmaxdatemonth_365T', 'overdueamountmaxdateyear_2T', \n                   'overdueamountmaxdateyear_994T', 'periodicityofpmts_1102L', 'periodicityofpmts_837L', 'prolongationcount_1120L', \n                   ]\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(24, 20))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_a_1_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:48:28.245966Z","iopub.execute_input":"2024-02-12T12:48:28.246872Z","iopub.status.idle":"2024-02-12T12:48:37.470550Z","shell.execute_reply.started":"2024-02-12T12:48:28.246841Z","shell.execute_reply":"2024-02-12T12:48:37.469571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ \n                   'prolongationcount_599L', 'purposeofcred_426M', 'purposeofcred_874M', 'refreshdate_3813885D', \n                   'residualamount_488A', 'residualamount_856A', 'subjectrole_182M', 'subjectrole_93M', \n                   'totalamount_6A', 'totalamount_996A', 'totaldebtoverduevalue_178A', 'totaldebtoverduevalue_718A', \n                   'totaloutstanddebtvalue_39A', 'totaloutstanddebtvalue_668A']\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(24, 20))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_a_1_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:49:08.532066Z","iopub.execute_input":"2024-02-12T12:49:08.532441Z","iopub.status.idle":"2024-02-12T12:49:12.836728Z","shell.execute_reply.started":"2024-02-12T12:49:08.532414Z","shell.execute_reply":"2024-02-12T12:49:12.836045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata9 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_credit_bureau_a_2_1.parquet')\n\nprint(data9.shape)\n# Merge the dataframes on 'case_id'\ndata9_df = data9.to_pandas()\ndata9_df = data9_df.sample(frac=0.2) # reduce data to manage\ntrain_credit_bureau_a_2_1_df = pd.merge(data9_df, base_data, on='case_id', how='left')\n\nprint(train_credit_bureau_a_2_1_df.shape)\ntrain_credit_bureau_a_2_1_df.dropna(subset=['collater_typofvalofguarant_298M'], inplace=True)\nprint(train_credit_bureau_a_2_1_df.shape)\nprint(train_credit_bureau_a_2_1_df.columns)\n\ntrain_credit_bureau_a_2_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:49:35.009641Z","iopub.execute_input":"2024-02-12T12:49:35.010000Z","iopub.status.idle":"2024-02-12T12:49:40.096557Z","shell.execute_reply.started":"2024-02-12T12:49:35.009971Z","shell.execute_reply":"2024-02-12T12:49:40.095604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_credit_bureau_a_2_1_df_forplot = train_credit_bureau_a_2_1_df.sample(frac=0.02)  \n\ncolumns_to_plot = ['collater_typofvalofguarant_298M', 'collater_typofvalofguarant_407M', 'collater_valueofguarantee_1124L', \n                   'collater_valueofguarantee_876L', 'collaterals_typeofguarante_359M', 'collaterals_typeofguarante_669M', 'num_group1', \n                   'num_group2', 'pmts_dpd_1073P', 'pmts_dpd_303P', 'pmts_month_158T', \n                   'pmts_month_706T', 'pmts_overdue_1140A', 'pmts_overdue_1152A', 'pmts_year_1139T', \n                   'pmts_year_507T', 'subjectroles_name_541M', 'subjectroles_name_838M']\n\nfig, axes = plt.subplots(nrows=5, ncols=4, figsize=(24, 20))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_credit_bureau_a_2_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:50:04.802015Z","iopub.execute_input":"2024-02-12T12:50:04.802371Z","iopub.status.idle":"2024-02-12T12:50:14.638005Z","shell.execute_reply.started":"2024-02-12T12:50:04.802344Z","shell.execute_reply":"2024-02-12T12:50:14.636991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata10 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_a_1.parquet')\nprint(data10.shape)\n# Merge the dataframes on 'case_id'\ndata10_df = data10.to_pandas()\ndata10_df = data10_df.sample(frac=0.5) #for memory control\ntrain_tax_registry_a_1_df = pd.merge(data10_df, base_data, on='case_id')\n\nprint(train_tax_registry_a_1_df.shape)\ntrain_tax_registry_a_1_df.dropna(subset=['amount_4527230A'], inplace=True)\nprint(train_tax_registry_a_1_df.shape)\nprint(train_tax_registry_a_1_df.columns)\n\ntrain_tax_registry_a_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:50:30.543246Z","iopub.execute_input":"2024-02-12T12:50:30.543564Z","iopub.status.idle":"2024-02-12T12:50:32.480595Z","shell.execute_reply.started":"2024-02-12T12:50:30.543540Z","shell.execute_reply":"2024-02-12T12:50:32.479767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tax_registry_a_1_df_forplot = train_tax_registry_a_1_df.sample(frac=0.03)  \n\ncolumns_to_plot = ['amount_4527230A', 'name_4527232M', 'num_group1', 'recorddate_4527225D']\n\nfig, axes = plt.subplots(nrows=2, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_tax_registry_a_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:50:42.865438Z","iopub.execute_input":"2024-02-12T12:50:42.865764Z","iopub.status.idle":"2024-02-12T12:53:12.588247Z","shell.execute_reply.started":"2024-02-12T12:50:42.865740Z","shell.execute_reply":"2024-02-12T12:53:12.587329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata11 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_b_1.parquet')\nprint(data11.shape)\n# Merge the dataframes on 'case_id'\ndata11_df = data11.to_pandas()\ntrain_tax_registry_b_1_df = pd.merge(data11_df, base_data, on='case_id', how='left')\n\nprint(train_tax_registry_b_1_df.shape)\ntrain_tax_registry_b_1_df.dropna(subset=['amount_4917619A'], inplace=True)\nprint(train_tax_registry_b_1_df.shape)\nprint(train_tax_registry_b_1_df.columns)\n\ntrain_tax_registry_b_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:53:33.869969Z","iopub.execute_input":"2024-02-12T12:53:33.870355Z","iopub.status.idle":"2024-02-12T12:53:36.214479Z","shell.execute_reply.started":"2024-02-12T12:53:33.870326Z","shell.execute_reply":"2024-02-12T12:53:36.213846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tax_registry_b_1_df_forplot = train_tax_registry_b_1_df.sample(frac=0.02)  \n\ncolumns_to_plot = ['amount_4917619A', 'deductiondate_4917603D', 'name_4917606M', 'num_group1']\n\nfig, axes = plt.subplots(nrows=2, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_tax_registry_b_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:53:49.084508Z","iopub.execute_input":"2024-02-12T12:53:49.084818Z","iopub.status.idle":"2024-02-12T12:55:00.694412Z","shell.execute_reply.started":"2024-02-12T12:53:49.084795Z","shell.execute_reply":"2024-02-12T12:55:00.693556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata12 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_tax_registry_c_1.parquet')\nprint(data12.shape)\n# Merge the dataframes on 'case_id'\ndata12_df = data12.to_pandas()\ndata12_df = data12_df.sample(frac=0.5) #for memory control\ntrain_tax_registry_c_1_df = pd.merge(data12_df, base_data, on='case_id')\n\nprint(train_tax_registry_c_1_df.shape)\ntrain_tax_registry_c_1_df.dropna(subset=['pmtamount_36A'], inplace=True)\nprint(train_tax_registry_c_1_df.shape)\nprint(train_tax_registry_c_1_df.columns)\n\ntrain_tax_registry_c_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:55:34.097944Z","iopub.execute_input":"2024-02-12T12:55:34.098335Z","iopub.status.idle":"2024-02-12T12:55:37.141851Z","shell.execute_reply.started":"2024-02-12T12:55:34.098307Z","shell.execute_reply":"2024-02-12T12:55:37.140974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tax_registry_c_1_df_forplot = train_tax_registry_c_1_df.sample(frac=0.02)  \n\ncolumns_to_plot = ['employername_160M', 'num_group1', 'pmtamount_36A', 'processingdate_168D']\n\nfig, axes = plt.subplots(nrows=2, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_tax_registry_c_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T12:55:57.284364Z","iopub.execute_input":"2024-02-12T12:55:57.284717Z","iopub.status.idle":"2024-02-12T12:57:51.357057Z","shell.execute_reply.started":"2024-02-12T12:55:57.284688Z","shell.execute_reply":"2024-02-12T12:57:51.355992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata13 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet')\nprint(data13.shape)\n\n# Merge the dataframes on 'case_id'\ntrain_data_df = data13.to_pandas()\nprint(train_data_df.columns)\n\nprint(train_data_df.shape)\n# train_data_df.dropna(subset=['WEEK_NUM'], inplace=True)\nprint(train_data_df.shape)\nprint(train_data_df.columns)\n\ntrain_base_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:16:59.714468Z","iopub.execute_input":"2024-02-12T13:16:59.714895Z","iopub.status.idle":"2024-02-12T13:17:00.179616Z","shell.execute_reply.started":"2024-02-12T13:16:59.714859Z","shell.execute_reply":"2024-02-12T13:17:00.178630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df.columns","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:19:01.465432Z","iopub.execute_input":"2024-02-12T13:19:01.468405Z","iopub.status.idle":"2024-02-12T13:19:01.475582Z","shell.execute_reply.started":"2024-02-12T13:19:01.468363Z","shell.execute_reply":"2024-02-12T13:19:01.474300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base_df_forplot = train_data_df.sample(frac=0.02)  \n\ncolumns_to_plot = ['date_decision', 'MONTH', 'WEEK_NUM']\n\nfig, axes = plt.subplots(nrows=2, ncols=2, figsize=(8, 8))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_base_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:20:19.418473Z","iopub.execute_input":"2024-02-12T13:20:19.418829Z","iopub.status.idle":"2024-02-12T13:20:24.944722Z","shell.execute_reply.started":"2024-02-12T13:20:19.418802Z","shell.execute_reply":"2024-02-12T13:20:24.943678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata14 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_person_1.parquet')\nprint(data14.shape)\n# Merge the dataframes on 'case_id'\ndata14_df = data14.to_pandas()\ntrain_person_1_df = pd.merge(data14_df, base_data, on='case_id')\n\nprint(train_person_1_df.shape)\ntrain_person_1_df.dropna(subset=['birthdate_87D'], inplace=True)\nprint(train_person_1_df.shape)\nprint(train_person_1_df.columns)\n\ntrain_person_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:08:24.165939Z","iopub.execute_input":"2024-02-12T13:08:24.166306Z","iopub.status.idle":"2024-02-12T13:08:28.378345Z","shell.execute_reply.started":"2024-02-12T13:08:24.166277Z","shell.execute_reply":"2024-02-12T13:08:28.377387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_person_1_df_forplot = train_person_1_df.sample(frac=0.2)  \n\ncolumns_to_plot = ['birth_259D', 'birthdate_87D', 'childnum_185L', 'contaddr_district_15M', \n                   'contaddr_matchlist_1032L', 'contaddr_smempladdr_334L', 'contaddr_zipcode_807M', 'education_927M', \n                   'empl_employedfrom_271D', 'empl_employedtotal_800L', 'empl_industry_691L', 'empladdr_district_926M', \n                   'empladdr_zipcode_114M', 'familystate_447L', 'gender_992L', 'housetype_905L', \n                   'housingtype_772L', 'incometype_1044T', 'isreference_387L', 'language1_981M', ]\n\nfig, axes = plt.subplots(nrows=5, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_person_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:08:42.171251Z","iopub.execute_input":"2024-02-12T13:08:42.171596Z","iopub.status.idle":"2024-02-12T13:08:59.610968Z","shell.execute_reply.started":"2024-02-12T13:08:42.171570Z","shell.execute_reply":"2024-02-12T13:08:59.609946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = ['mainoccupationinc_384A', 'maritalst_703L', 'num_group1', 'personindex_1023L', \n                   'persontype_1072L', 'persontype_792L', 'registaddr_district_1083M', 'registaddr_zipcode_184M', \n                   'relationshiptoclient_415T', 'relationshiptoclient_642T', 'remitter_829L', 'role_1084L', \n                   'role_993L', 'safeguarantyflag_411L', 'sex_738L', 'type_25L']\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_person_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:09:23.979479Z","iopub.execute_input":"2024-02-12T13:09:23.979837Z","iopub.status.idle":"2024-02-12T13:09:32.352342Z","shell.execute_reply.started":"2024-02-12T13:09:23.979811Z","shell.execute_reply":"2024-02-12T13:09:32.351060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata15 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_deposit_1.parquet')\nprint(data15.shape)\n# Merge the dataframes on 'case_id'\ndata15_df = data15.to_pandas()\ntrain_deposit_1_df = pd.merge(data15_df, base_data, on='case_id')\n\nprint(train_deposit_1_df.shape)\ntrain_deposit_1_df.dropna(subset=['amount_416A'], inplace=True)\nprint(train_deposit_1_df.shape)\nprint(train_deposit_1_df.columns)\n\ntrain_deposit_1_df.isnull().sum()\n\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:09:43.513468Z","iopub.execute_input":"2024-02-12T13:09:43.513808Z","iopub.status.idle":"2024-02-12T13:09:43.997068Z","shell.execute_reply.started":"2024-02-12T13:09:43.513781Z","shell.execute_reply":"2024-02-12T13:09:43.996147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_deposit_1_df_forplot = train_deposit_1_df.sample(frac=0.02)  \n\ncolumns_to_plot = ['amount_416A', 'contractenddate_991D', 'num_group1', 'openingdate_313D']\n\nfig, axes = plt.subplots(nrows=2, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_deposit_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:09:50.478402Z","iopub.execute_input":"2024-02-12T13:09:50.478793Z","iopub.status.idle":"2024-02-12T13:09:59.094418Z","shell.execute_reply.started":"2024-02-12T13:09:50.478742Z","shell.execute_reply":"2024-02-12T13:09:59.093213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata16 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_debitcard_1.parquet')\nprint(data16.shape)\n# Merge the dataframes on 'case_id'\ndata16_df = data16.to_pandas()\ntrain_debitcard_1_df = pd.merge(data16_df, base_data, on='case_id')\n\nprint(train_debitcard_1_df.shape)\ntrain_debitcard_1_df.dropna(subset=['last180dayaveragebalance_704A'], inplace=True)\nprint(train_debitcard_1_df.shape)\nprint(train_debitcard_1_df.columns)\n\ntrain_debitcard_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:10:09.955229Z","iopub.execute_input":"2024-02-12T13:10:09.955576Z","iopub.status.idle":"2024-02-12T13:10:10.408260Z","shell.execute_reply.started":"2024-02-12T13:10:09.955550Z","shell.execute_reply":"2024-02-12T13:10:10.407413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_debitcard_1_df_forplot = train_debitcard_1_df.sample(frac=0.5)  \n\ncolumns_to_plot = [ 'last180dayaveragebalance_704A', 'last180dayturnover_1134A', 'last30dayturnover_651A', 'num_group1', 'openingdate_857D']\n\nfig, axes = plt.subplots(nrows=3, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_debitcard_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:10:19.515613Z","iopub.execute_input":"2024-02-12T13:10:19.515963Z","iopub.status.idle":"2024-02-12T13:10:22.773437Z","shell.execute_reply.started":"2024-02-12T13:10:19.515937Z","shell.execute_reply":"2024-02-12T13:10:22.772350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata17 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_other_1.parquet')\nprint(data17.shape)\n# Merge the dataframes on 'case_id'\ndata17_df = data17.to_pandas()\ntrain_other_1_df = pd.merge(data17_df, base_data, on='case_id')\n\nprint(train_other_1_df.shape)\ntrain_other_1_df.dropna(subset=['amtdebitincoming_4809443A'], inplace=True)\nprint(train_other_1_df.shape)\nprint(train_other_1_df.columns)\n\ntrain_other_1_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:10:41.665336Z","iopub.execute_input":"2024-02-12T13:10:41.665652Z","iopub.status.idle":"2024-02-12T13:10:41.994455Z","shell.execute_reply.started":"2024-02-12T13:10:41.665626Z","shell.execute_reply":"2024-02-12T13:10:41.993512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_other_1_df_forplot = train_other_1_df.sample(frac=0.05)  \n\ncolumns_to_plot = [ 'amtdebitincoming_4809443A', 'amtdebitoutgoing_4809440A', 'amtdepositbalance_4809441A', 'amtdepositincoming_4809444A', 'amtdepositoutgoing_4809442A', 'num_group1']\n\nfig, axes = plt.subplots(nrows=3, ncols=2, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_other_1_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:11:01.034535Z","iopub.execute_input":"2024-02-12T13:11:01.034870Z","iopub.status.idle":"2024-02-12T13:11:02.856186Z","shell.execute_reply.started":"2024-02-12T13:11:01.034843Z","shell.execute_reply":"2024-02-12T13:11:02.855017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ndata18 = pq.read_table('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_static_0_0.parquet')\nprint(data18.shape)\n# Merge the dataframes on 'case_id'\ndata18_df = data18.to_pandas()\ntrain_static_0_0_df = pd.merge(data18_df, base_data, on='case_id')\n\nprint(train_static_0_0_df.shape)\n# train_static_0_0_df.dropna(subset=['actualdpdtolerance_344P'], inplace=True)\nprint(train_static_0_0_df.shape)\nprint(train_static_0_0_df.columns)\n\ntrain_static_0_0_df.isnull().sum()\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:12:45.820831Z","iopub.execute_input":"2024-02-12T13:12:45.821210Z","iopub.status.idle":"2024-02-12T13:12:49.484503Z","shell.execute_reply.started":"2024-02-12T13:12:45.821180Z","shell.execute_reply":"2024-02-12T13:12:49.483163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0_0_df_forplot = train_static_0_0_df.sample(frac=0.005)  \n\ncolumns_to_plot = [ 'actualdpdtolerance_344P', 'amtinstpaidbefduel24m_4187115A', 'annuity_780A', 'annuitynextmonth_57A', \n                   'applicationcnt_361L', 'applications30d_658L', 'applicationscnt_1086L', 'applicationscnt_464L', \n                   'applicationscnt_629L', 'applicationscnt_867L', 'avgdbddpdlast24m_3658932P', 'avgdbddpdlast3m_4187120P', \n                   'avgdbdtollast24m_4525197P', 'avgdpdtolclosure24_3658938P', 'avginstallast24m_3658937A', 'avglnamtstart24m_4525187A', \n                   'avgmaxdpdlast9m_3716943P', 'avgoutstandbalancel6m_4187114A', 'avgpmtlast12m_4525200A', 'bankacctype_710L', \n                   'cardtype_51L', 'clientscnt12m_3712952L', 'clientscnt3m_3712950L', 'clientscnt6m_3712949L', ]\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:13:06.765833Z","iopub.execute_input":"2024-02-12T13:13:06.766215Z","iopub.status.idle":"2024-02-12T13:13:13.355268Z","shell.execute_reply.started":"2024-02-12T13:13:06.766190Z","shell.execute_reply":"2024-02-12T13:13:13.354651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ 'clientscnt_100L', 'clientscnt_1022L', 'clientscnt_1071L', 'clientscnt_1130L', \n                   'clientscnt_136L', 'clientscnt_157L', 'clientscnt_257L', 'clientscnt_304L', \n                   'clientscnt_360L', 'clientscnt_493L', 'clientscnt_533L', 'clientscnt_887L', \n                   'clientscnt_946L', 'cntincpaycont9m_3716944L', 'cntpmts24_3658933L', 'commnoinclast6m_3546845L', \n                   'credamount_770A', 'credtype_322L', 'currdebt_22A', 'currdebtcredtyperange_828A', \n                   'datefirstoffer_1144D', 'datelastinstal40dpd_247D', 'datelastunpaid_3546854D', 'daysoverduetolerancedd_3976961L', ]\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:13:25.235582Z","iopub.execute_input":"2024-02-12T13:13:25.235887Z","iopub.status.idle":"2024-02-12T13:13:41.766445Z","shell.execute_reply.started":"2024-02-12T13:13:25.235864Z","shell.execute_reply":"2024-02-12T13:13:41.765523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = ['deferredmnthsnum_166L', 'disbursedcredamount_1113A', 'disbursementtype_67L', 'downpmt_116A', \n                   'dtlastpmtallstes_4499206D', 'eir_270L', 'equalitydataagreement_891L', 'equalityempfrom_62L', \n                   'firstclxcampaign_1125D', 'firstdatedue_489D', 'homephncnt_628L', 'inittransactionamount_650A', \n                   'inittransactioncode_186L', 'interestrate_311L', 'interestrategrace_34L', 'isbidproduct_1095L', \n                   'isbidproductrequest_292L', 'isdebitcard_729L', 'lastactivateddate_801D', 'lastapplicationdate_877D', \n                   'lastapprcommoditycat_1041M', 'lastapprcommoditytypec_5251766M', 'lastapprcredamount_781A', 'lastapprdate_640D', ]\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:14:04.803086Z","iopub.execute_input":"2024-02-12T13:14:04.803444Z","iopub.status.idle":"2024-02-12T13:14:44.623126Z","shell.execute_reply.started":"2024-02-12T13:14:04.803417Z","shell.execute_reply":"2024-02-12T13:14:44.622353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ 'lastcancelreason_561M', 'lastdelinqdate_224D', 'lastdependentsnum_448L', 'lastotherinc_902A', \n                   'lastotherlnsexpense_631A', 'lastrejectcommoditycat_161M', 'lastrejectcommodtypec_5251769M', 'lastrejectcredamount_222A', \n                   'lastrejectdate_50D', 'lastrejectreason_759M', 'lastrejectreasonclient_4145040M', 'lastrepayingdate_696D', \n                   'lastst_736L', 'maininc_215A', 'mastercontrelectronic_519L', 'mastercontrexist_109L', \n                   'maxannuity_159A', 'maxannuity_4075009A', 'maxdbddpdlast1m_3658939P', 'maxdbddpdtollast12m_3658940P', \n                   'maxdbddpdtollast6m_4187119P', 'maxdebt4_972A', 'maxdpdfrom6mto36m_3546853P', 'maxdpdinstldate_3546855D', ]\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:15:02.441390Z","iopub.execute_input":"2024-02-12T13:15:02.441719Z","iopub.status.idle":"2024-02-12T13:15:27.109844Z","shell.execute_reply.started":"2024-02-12T13:15:02.441693Z","shell.execute_reply":"2024-02-12T13:15:27.108808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ 'maxdpdinstlnum_3546846P', 'maxdpdlast12m_727P', 'maxdpdlast24m_143P', 'maxdpdlast3m_392P', \n                   'maxdpdlast6m_474P', 'maxdpdlast9m_1059P', 'maxdpdtolerance_374P', 'maxinstallast24m_3658928A', \n                   'maxlnamtstart6m_4525199A', 'maxoutstandbalancel12m_4187113A', 'maxpmtlast3m_4525190A', 'mindbddpdlast24m_3658935P', \n                   'mindbdtollast24m_4525191P', 'mobilephncnt_593L', 'monthsannuity_845L', 'numactivecreds_622L', \n                   'numactivecredschannel_414L', 'numactiverelcontr_750L', 'numcontrs3months_479L', 'numincomingpmts_3546848L', \n                   'numinstlallpaidearly3d_817L', 'numinstls_657L', 'numinstlsallpaid_934L', 'numinstlswithdpd10_728L', ]\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:15:57.205263Z","iopub.execute_input":"2024-02-12T13:15:57.205602Z","iopub.status.idle":"2024-02-12T13:16:04.235776Z","shell.execute_reply.started":"2024-02-12T13:15:57.205577Z","shell.execute_reply":"2024-02-12T13:16:04.234603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ 'numinstlswithdpd5_4187116L', 'numinstlswithoutdpd_562L', 'numinstmatpaidtearly2d_4499204L', 'numinstpaid_4499208L', \n                   'numinstpaidearly3d_3546850L', 'numinstpaidearly3dest_4493216L', 'numinstpaidearly5d_1087L', 'numinstpaidearly5dest_4493211L', \n                   'numinstpaidearly5dobd_4499205L', 'numinstpaidearly_338L', 'numinstpaidearlyest_4493214L', 'numinstpaidlastcontr_4325080L', \n                   'numinstpaidlate1d_3546852L', 'numinstregularpaid_973L', 'numinstregularpaidest_4493210L', 'numinsttopaygr_769L', \n                   'numinsttopaygrest_4493213L', 'numinstunpaidmax_3546851L', 'numinstunpaidmaxest_4493212L', 'numnotactivated_1143L', \n                   'numpmtchanneldd_318L', 'numrejects9m_859L', 'opencred_647L', 'paytype1st_925L', ]\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:16:11.710389Z","iopub.execute_input":"2024-02-12T13:16:11.710712Z","iopub.status.idle":"2024-02-12T13:16:18.122596Z","shell.execute_reply.started":"2024-02-12T13:16:11.710688Z","shell.execute_reply":"2024-02-12T13:16:18.121196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_plot = [ 'paytype_783L', 'payvacationpostpone_4187118D', 'pctinstlsallpaidearl3d_427L', 'pctinstlsallpaidlat10d_839L', \n                   'pctinstlsallpaidlate1d_3546856L', 'pctinstlsallpaidlate4d_3546849L', 'pctinstlsallpaidlate6d_3546844L', 'pmtnum_254L', \n                   'posfpd10lastmonth_333P', 'posfpd30lastmonth_3976960P', 'posfstqpd30lastmonth_3976962P', 'previouscontdistrict_112M', \n                   'price_1097A', 'sellerplacecnt_915L', 'sellerplacescnt_216L', 'sumoutstandtotal_3546847A', \n                   'sumoutstandtotalest_4493215A', 'totaldebt_9A', 'totalsettled_863A', 'totinstallast1m_4525188A', \n                   'twobodfilling_608L', 'typesuite_864L', 'validfrom_1069D']\n\nfig, axes = plt.subplots(nrows=6, ncols=4, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flat):\n    if i < len(columns_to_plot):\n        sns.scatterplot(data=train_static_0_0_df_forplot, x=columns_to_plot[i], y='target', hue='target', ax=ax, alpha=0.5)\n        ax.set_title(f'{columns_to_plot[i]} with default')\n        ax.set_xlabel('Feature Value')\n        ax.set_ylabel('default')\n        ax.legend(title='target', loc='upper right')\n\n# Hide any empty subplots\nfor ax in axes.flat[len(columns_to_plot):]:\n    ax.axis('off')\n\nplt.tight_layout()\nplt.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-02-12T13:16:24.890910Z","iopub.execute_input":"2024-02-12T13:16:24.891259Z","iopub.status.idle":"2024-02-12T13:16:33.221635Z","shell.execute_reply.started":"2024-02-12T13:16:24.891233Z","shell.execute_reply":"2024-02-12T13:16:33.220478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# import os\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         p = os.path.join(dirname, filename)\n# #         print(p)\n#         if filename.endswith('.parquet'):\n#             continue\n#         else:\n#             try:\n#                 a = pd.read_csv(p)\n#                 print(filename, a.shape)\n#             except pd.errors.EmptyDataError:\n#                 print(f\"Empty or invalid CSV file: {p}\")\n#             except pd.errors.ParserError:\n#                 print(f\"Error parsing CSV file: {p}\")\n        ","metadata":{"execution":{"iopub.status.busy":"2024-02-08T15:48:33.378117Z","iopub.execute_input":"2024-02-08T15:48:33.378600Z","iopub.status.idle":"2024-02-08T15:48:33.383313Z","shell.execute_reply.started":"2024-02-08T15:48:33.378566Z","shell.execute_reply":"2024-02-08T15:48:33.382402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import pandas as pd\n\n# result_data = []  # List to store data for each file\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         p = os.path.join(dirname, filename)\n#         if filename.endswith('.parquet'):\n#             continue\n#         else:\n#             try:\n#                 df = pd.read_csv(p)\n#                 row_data = {'Filename': filename, 'Shape': df.shape, 'Index': df.columns.tolist()}\n#                 result_data.append(row_data)\n#                 print(filename, df.shape, df.columns.tolist())\n#             except pd.errors.EmptyDataError:\n#                 print(f\"Empty or invalid CSV file: {p}\")\n#             except pd.errors.ParserError:\n#                 print(f\"Error parsing CSV file: {p}\")\n\n# # Convert list of dictionaries to a DataFrame\n# result_df = pd.DataFrame(result_data)\n\n# # Convert DataFrame to JSON file\n# result_df.to_json('output.json', orient='records')","metadata":{"execution":{"iopub.status.busy":"2024-02-08T15:48:33.385072Z","iopub.execute_input":"2024-02-08T15:48:33.386017Z","iopub.status.idle":"2024-02-08T16:04:56.285154Z","shell.execute_reply.started":"2024-02-08T15:48:33.385986Z","shell.execute_reply":"2024-02-08T16:04:56.284141Z"},"trusted":true},"execution_count":null,"outputs":[]}]}