{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7602123,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:56:09.812071Z","iopub.execute_input":"2024-02-23T14:56:09.812743Z","iopub.status.idle":"2024-02-23T14:56:09.819842Z","shell.execute_reply.started":"2024-02-23T14:56:09.812691Z","shell.execute_reply":"2024-02-23T14:56:09.818362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature_def = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:56:09.868572Z","iopub.execute_input":"2024-02-23T14:56:09.870048Z","iopub.status.idle":"2024-02-23T14:56:09.882201Z","shell.execute_reply.started":"2024-02-23T14:56:09.869948Z","shell.execute_reply":"2024-02-23T14:56:09.880948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature_def[df_feature_def['Variable']=='opencred_647L']","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:56:09.887976Z","iopub.execute_input":"2024-02-23T14:56:09.888535Z","iopub.status.idle":"2024-02-23T14:56:09.901226Z","shell.execute_reply.started":"2024-02-23T14:56:09.888495Z","shell.execute_reply":"2024-02-23T14:56:09.899926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selects only depth=0 train tables\n\n# base\ndf_train_base = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv')\n\n# statics\ndf_train_static_0_0 = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_0.csv')\ndf_train_static_0_1 = pd.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_1.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:56:09.933378Z","iopub.execute_input":"2024-02-23T14:56:09.934629Z","iopub.status.idle":"2024-02-23T14:57:03.818987Z","shell.execute_reply.started":"2024-02-23T14:56:09.934583Z","shell.execute_reply":"2024-02-23T14:57:03.814696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_base.shape)\nprint(df_train_base.head(2))","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:03.829605Z","iopub.execute_input":"2024-02-23T14:57:03.832517Z","iopub.status.idle":"2024-02-23T14:57:03.86011Z","shell.execute_reply.started":"2024-02-23T14:57:03.832207Z","shell.execute_reply":"2024-02-23T14:57:03.856337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_static_0_1.shape)\nprint(df_train_static_0_0.shape)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:03.865397Z","iopub.execute_input":"2024-02-23T14:57:03.866661Z","iopub.status.idle":"2024-02-23T14:57:03.885355Z","shell.execute_reply.started":"2024-02-23T14:57:03.866564Z","shell.execute_reply":"2024-02-23T14:57:03.880286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concat train static tables\ndf_train_static = pd.concat([df_train_static_0_0,df_train_static_0_1])\n\n# merge basetable with static table\ndf = pd.merge(df_train_base,df_train_static,how='left',on='case_id')\n\n# create final dataframe\n# df = pd.merge(df,df_train_static_cb_0,how='left',on='case_id')","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:03.894294Z","iopub.execute_input":"2024-02-23T14:57:03.896366Z","iopub.status.idle":"2024-02-23T14:57:28.304165Z","shell.execute_reply.started":"2024-02-23T14:57:03.896263Z","shell.execute_reply":"2024-02-23T14:57:28.302919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_cols = [ 'case_id','date_decision','MONTH','WEEK_NUM','target']\nsel_non_imputated_cols = ['actualdpdtolerance_344P','amtinstpaidbefduel24m_4187115A','annuity_780A','annuitynextmonth_57A','applicationcnt_361L','applications30d_658L','applicationscnt_1086L','applicationscnt_464L','applicationscnt_629L','applicationscnt_867L','avgdbddpdlast24m_3658932P','avgdbddpdlast3m_4187120P','avgdbdtollast24m_4525197P','avgdpdtolclosure24_3658938P','avginstallast24m_3658937A','avgmaxdpdlast9m_3716943P','avgoutstandbalancel6m_4187114A','avgpmtlast12m_4525200A','clientscnt_100L','clientscnt_1022L','clientscnt_1071L','clientscnt_1130L','clientscnt_157L','clientscnt_257L','clientscnt_304L','clientscnt_360L','clientscnt_493L','clientscnt_533L','clientscnt_887L','clientscnt_946L','clientscnt12m_3712952L','clientscnt3m_3712950L','clientscnt6m_3712949L','credamount_770A','credtype_322L','currdebt_22A','currdebtcredtyperange_828A','daysoverduetolerancedd_3976961L','deferredmnthsnum_166L','disbursedcredamount_1113A','disbursementtype_67L','downpmt_116A','isbidproduct_1095L','lastapprcommoditycat_1041M','lastapprcommoditytypec_5251766M','lastcancelreason_561M','lastrejectcommoditycat_161M','lastrejectcommodtypec_5251769M','lastrejectreason_759M','lastrejectreasonclient_4145040M','mobilephncnt_593L','numactivecreds_622L','numactivecredschannel_414L','numactiverelcontr_750L','numcontrs3months_479L','numnotactivated_1143L','numpmtchanneldd_318L','numrejects9m_859L','previouscontdistrict_112M','sellerplacecnt_915L','sellerplacescnt_216L']\nsel_imputated_cols = ['bankacctype_710L','cardtype_51L','cntincpaycont9m_3716944L','cntpmts24_3658933L','commnoinclast6m_3546845L','eir_270L','inittransactioncode_186L','interestrate_311L','lastapprcredamount_781A','lastrejectcredamount_222A','lastst_736L','maininc_215A','maxannuity_159A','maxdbddpdtollast6m_4187119P','numinstls_657L','numinstlsallpaid_934L','numinstpaidearly_338L','numinstregularpaid_973L','numinsttopaygr_769L','numinstunpaidmax_3546851L','opencred_647L','pmtnum_254L','posfpd30lastmonth_3976960P','price_1097A','sumoutstandtotal_3546847A','totaldebt_9A','totalsettled_863A']\nimputate_with_zero_cols = ['daysoverduetolerancedd_3976961L','avgpmtlast12m_4525200A','avgoutstandbalancel6m_4187114A','avgmaxdpdlast9m_3716943P','avginstallast24m_3658937A','avgdpdtolclosure24_3658938P','avgdbdtollast24m_4525197P','avgdbddpdlast3m_4187120P','avgdbddpdlast24m_3658932P','amtinstpaidbefduel24m_4187115A','actualdpdtolerance_344P','cntincpaycont9m_3716944L','cntpmts24_3658933L','commnoinclast6m_3546845L','lastapprcredamount_781A','lastrejectcredamount_222A','maininc_215A','maxannuity_159A','numinstls_657L','numinstlsallpaid_934L','numinstpaidearly_338L','numinstregularpaid_973L','numinsttopaygr_769L','numinstunpaidmax_3546851L','pmtnum_254L','posfpd30lastmonth_3976960P','price_1097A','sumoutstandtotal_3546847A','totaldebt_9A','totalsettled_863A']","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:28.305888Z","iopub.execute_input":"2024-02-23T14:57:28.306248Z","iopub.status.idle":"2024-02-23T14:57:28.320582Z","shell.execute_reply.started":"2024-02-23T14:57:28.30622Z","shell.execute_reply":"2024-02-23T14:57:28.318372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def impute_with_zero(column):\n    df[column] = df[column].fillna(0.0)\n    \n# imputation with zero\nimpute_with_zero(imputate_with_zero_cols)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:28.322539Z","iopub.execute_input":"2024-02-23T14:57:28.32304Z","iopub.status.idle":"2024-02-23T14:57:28.927031Z","shell.execute_reply.started":"2024-02-23T14:57:28.322991Z","shell.execute_reply":"2024-02-23T14:57:28.925774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:28.9286Z","iopub.execute_input":"2024-02-23T14:57:28.928973Z","iopub.status.idle":"2024-02-23T14:57:28.936408Z","shell.execute_reply.started":"2024-02-23T14:57:28.928941Z","shell.execute_reply":"2024-02-23T14:57:28.93515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(imputate_with_zero_cols)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:28.937775Z","iopub.execute_input":"2024-02-23T14:57:28.938137Z","iopub.status.idle":"2024-02-23T14:57:28.949301Z","shell.execute_reply.started":"2024-02-23T14:57:28.938099Z","shell.execute_reply":"2024-02-23T14:57:28.948109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# manual imputation\n\n# bankacctype_710L\ndf['bankacctype_710L'] = df['bankacctype_710L'].fillna('NA')\n\n# cardtype_51L\ndf['cardtype_51L'] = df['cardtype_51L'].fillna('NOCARD')\n\n# eir_270L\ndf['eir_270L'] = df['eir_270L'].fillna(0.2)\n\n# inittransactioncode_186L\ndf['inittransactioncode_186L'] = df['inittransactioncode_186L'].fillna(df['inittransactioncode_186L'].mode()[0])\n\n# interestrate_311L\ndf['interestrate_311L'] = df['interestrate_311L'].fillna(df['interestrate_311L'].mean())\n\n# lastst_736L\ndf['lastst_736L'] = df['lastst_736L'].fillna(df['lastst_736L'].mode()[0])\n\n# opencred_647L\ndf['opencred_647L'] = df['opencred_647L'].fillna(df['opencred_647L'].mode()[0])\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:28.951063Z","iopub.execute_input":"2024-02-23T14:57:28.951434Z","iopub.status.idle":"2024-02-23T14:57:30.602223Z","shell.execute_reply.started":"2024-02-23T14:57:28.951404Z","shell.execute_reply":"2024-02-23T14:57:30.601072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selected columns for the final dataframe\nfinal_cols = base_cols + sel_imputated_cols + sel_non_imputated_cols","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:30.605578Z","iopub.execute_input":"2024-02-23T14:57:30.60596Z","iopub.status.idle":"2024-02-23T14:57:30.611889Z","shell.execute_reply.started":"2024-02-23T14:57:30.605932Z","shell.execute_reply":"2024-02-23T14:57:30.610575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_frame = df[final_cols]","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:43.948881Z","iopub.execute_input":"2024-02-23T14:57:43.949329Z","iopub.status.idle":"2024-02-23T14:57:44.694385Z","shell.execute_reply.started":"2024-02-23T14:57:43.949297Z","shell.execute_reply":"2024-02-23T14:57:44.693305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_frame.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:45.445755Z","iopub.execute_input":"2024-02-23T14:57:45.446408Z","iopub.status.idle":"2024-02-23T14:57:45.452675Z","shell.execute_reply.started":"2024-02-23T14:57:45.446375Z","shell.execute_reply":"2024-02-23T14:57:45.451258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_frame.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:57:46.448632Z","iopub.execute_input":"2024-02-23T14:57:46.449083Z","iopub.status.idle":"2024-02-23T14:57:46.49001Z","shell.execute_reply.started":"2024-02-23T14:57:46.449049Z","shell.execute_reply":"2024-02-23T14:57:46.48849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Clustering","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.cluster import KMeans\n\n# Assuming df is your DataFrame\n\n# Selecting features for clustering (excluding target variable)\nfeatures = cleaned_frame.drop(columns=['target', 'case_id', 'date_decision']).columns\n\n# Initializing empty list to store inertias\ninertias = []\n\n# Running KMeans with different numbers of clusters\nfor k in range(1, 11):\n    kmeans = KMeans(n_clusters=k, random_state=42)\n    kmeans.fit(cleaned_frame[features])\n    inertias.append(kmeans.inertia_)\n\n# Plotting the elbow curve\nplt.figure(figsize=(10, 6))\nplt.plot(range(1, 11), inertias, marker='o', linestyle='--')\nplt.title('Elbow Method')\nplt.xlabel('Number of Clusters')\nplt.ylabel('Inertia')\nplt.xticks(range(1, 11))\nplt.grid(True)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T15:18:31.558877Z","iopub.execute_input":"2024-02-23T15:18:31.559372Z","iopub.status.idle":"2024-02-23T15:18:35.502761Z","shell.execute_reply.started":"2024-02-23T15:18:31.559339Z","shell.execute_reply":"2024-02-23T15:18:35.501012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import silhouette_score\n","metadata":{"execution":{"iopub.status.busy":"2024-02-23T15:14:15.448757Z","iopub.execute_input":"2024-02-23T15:14:15.449239Z","iopub.status.idle":"2024-02-23T15:14:15.770268Z","shell.execute_reply.started":"2024-02-23T15:14:15.449208Z","shell.execute_reply":"2024-02-23T15:14:15.768905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocessing pipeline\nnumeric_features = cleaned_frame.select_dtypes(include=['float64', 'int64']).columns\ncategorical_features = cleaned_frame.select_dtypes(include=['object', 'bool']).columns","metadata":{"execution":{"iopub.status.busy":"2024-02-23T15:14:39.82356Z","iopub.execute_input":"2024-02-23T15:14:39.824233Z","iopub.status.idle":"2024-02-23T15:14:44.008006Z","shell.execute_reply.started":"2024-02-23T15:14:39.824191Z","shell.execute_reply":"2024-02-23T15:14:44.006678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(numeric_features)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T15:14:56.878534Z","iopub.execute_input":"2024-02-23T15:14:56.878934Z","iopub.status.idle":"2024-02-23T15:14:56.887087Z","shell.execute_reply.started":"2024-02-23T15:14:56.878905Z","shell.execute_reply":"2024-02-23T15:14:56.885773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(categorical_features)","metadata":{"execution":{"iopub.status.busy":"2024-02-23T15:15:04.148576Z","iopub.execute_input":"2024-02-23T15:15:04.14902Z","iopub.status.idle":"2024-02-23T15:15:04.157145Z","shell.execute_reply.started":"2024-02-23T15:15:04.148988Z","shell.execute_reply":"2024-02-23T15:15:04.155838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search_array(arr, target):\n    for item in arr:\n        if item == target:\n            return True\n    return False\n\nprint(search_array(df.columns, 'typesuite_864L'))  # Output: True","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:17:58.779637Z","iopub.execute_input":"2024-02-23T14:17:58.780057Z","iopub.status.idle":"2024-02-23T14:17:58.787089Z","shell.execute_reply.started":"2024-02-23T14:17:58.780028Z","shell.execute_reply":"2024-02-23T14:17:58.78579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def missing_handle(column_name):\n    print(search_array(df.columns, column_name)) \n    print('HEAD')\n    print(df[column_name].head(5))\n    print('')\n    print('INFO')\n    print(df[column_name].info())\n    print('')\n    print('DESCRIBE')\n    print(df[column_name].describe())\n    print('')\n    print('NULL Percentage')\n    print((df[column_name].isna().sum()/df[column_name].shape[0])*100)\n    print('')\n    print('MODE')\n    print(df[column_name].mode())\n    print('Unique')\n    print(df[column_name].unique())\n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:17:50.764247Z","iopub.execute_input":"2024-02-23T14:17:50.765357Z","iopub.status.idle":"2024-02-23T14:17:50.774852Z","shell.execute_reply.started":"2024-02-23T14:17:50.765313Z","shell.execute_reply":"2024-02-23T14:17:50.773432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_handle('maxdbddpdtollast6m_4187119P')","metadata":{"execution":{"iopub.status.busy":"2024-02-23T14:18:02.28847Z","iopub.execute_input":"2024-02-23T14:18:02.288908Z","iopub.status.idle":"2024-02-23T14:18:02.463109Z","shell.execute_reply.started":"2024-02-23T14:18:02.288876Z","shell.execute_reply":"2024-02-23T14:18:02.4618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['maxdbddpdtollast6m_4187119P'].isna()]['maxdbddpdtollast12m_3658940P'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:48:28.445039Z","iopub.execute_input":"2024-02-23T05:48:28.445456Z","iopub.status.idle":"2024-02-23T05:48:30.092232Z","shell.execute_reply.started":"2024-02-23T05:48:28.445425Z","shell.execute_reply":"2024-02-23T05:48:30.090943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['maxdbddpdtollast6m_4187119P'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T05:48:53.825145Z","iopub.execute_input":"2024-02-23T05:48:53.82625Z","iopub.status.idle":"2024-02-23T05:48:53.837337Z","shell.execute_reply.started":"2024-02-23T05:48:53.826209Z","shell.execute_reply":"2024-02-23T05:48:53.836097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-02-23T02:51:37.050204Z","iopub.execute_input":"2024-02-23T02:51:37.050715Z","iopub.status.idle":"2024-02-23T02:51:37.056634Z","shell.execute_reply.started":"2024-02-23T02:51:37.050681Z","shell.execute_reply":"2024-02-23T02:51:37.055306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot histogram\nplt.hist(df['price_1097A'])\n\n# Customize labels and title\nplt.xlabel('Values')\nplt.ylabel('Frequency')\nplt.title('Histogram of Column')\n\n# Show plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-23T02:56:26.020824Z","iopub.execute_input":"2024-02-23T02:56:26.021433Z","iopub.status.idle":"2024-02-23T02:56:26.31424Z","shell.execute_reply.started":"2024-02-23T02:56:26.021372Z","shell.execute_reply":"2024-02-23T02:56:26.312777Z"},"trusted":true},"execution_count":null,"outputs":[]}]}