{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore', category=RuntimeWarning)\n# Đọc dữ liệu từ file application\napp_data = pd.read_csv(r\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_static_0_0.csv\", low_memory=False)  # hoặc application_test.csv\n\ndata = app_data[['case_id','credtype_322L']]\ndata.value_counts()\ndata","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:25:46.354141Z","iopub.execute_input":"2025-04-07T14:25:46.354566Z","iopub.status.idle":"2025-04-07T14:26:32.738634Z","shell.execute_reply.started":"2025-04-07T14:25:46.354516Z","shell.execute_reply":"2025-04-07T14:26:32.737453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_1 = pd.read_csv(r\"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv\")  # hoặc application_test.csv\ndata_1 = data_1[['case_id','MONTH','target']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:26:32.740318Z","iopub.execute_input":"2025-04-07T14:26:32.740771Z","iopub.status.idle":"2025-04-07T14:26:33.894762Z","shell.execute_reply.started":"2025-04-07T14:26:32.740701Z","shell.execute_reply":"2025-04-07T14:26:33.893542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gộp 2 bảng theo case_id (inner join để chỉ giữ lại các case_id có trong cả 2 bảng)\nmerged_data = pd.merge(data, data_1, on='case_id', how='inner')\n\n# Xem kết quả\nmerged_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:26:33.897108Z","iopub.execute_input":"2025-04-07T14:26:33.897453Z","iopub.status.idle":"2025-04-07T14:26:33.956218Z","shell.execute_reply.started":"2025-04-07T14:26:33.897413Z","shell.execute_reply":"2025-04-07T14:26:33.955328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_rel = merged_data[merged_data['credtype_322L'] == 'REL'].reset_index(drop=True)\n\ndata_rel['MONTH'].value_counts().sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:42:06.403512Z","iopub.execute_input":"2025-04-07T14:42:06.404063Z","iopub.status.idle":"2025-04-07T14:42:06.5078Z","shell.execute_reply.started":"2025-04-07T14:42:06.404022Z","shell.execute_reply":"2025-04-07T14:42:06.506819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Bước 1: Chuyển cột MONTH sang kiểu chuỗi (string)\ndata_rel['MONTH'] = data_rel['MONTH'].astype(str)\n\n\ndata_rel","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:42:08.50985Z","iopub.execute_input":"2025-04-07T14:42:08.510203Z","iopub.status.idle":"2025-04-07T14:42:08.562499Z","shell.execute_reply.started":"2025-04-07T14:42:08.510177Z","shell.execute_reply":"2025-04-07T14:42:08.561086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def binning_all_variables(df, threshold=0.05, custom_bins=None, feature=None):\n    df = df.copy()\n\n    columns_to_process = [feature] if feature else df.columns\n\n    for column in columns_to_process:\n        if column == 'target':  # Bỏ qua cột target\n            continue\n\n        missing_mask = df[column].isna()\n\n        if df[column].dtype == 'O':  # Xử lý biến dạng chữ\n            df['GRP_' + column] = df[column].astype(object)\n            if missing_mask.any():\n                df.loc[missing_mask, 'GRP_' + column] = 'Missing'\n                add_missing = True\n            else:\n                add_missing = False\n        else:  # Xử lý biến số\n            if custom_bins and column in custom_bins:\n                bins = [-np.inf] + custom_bins[column] + [np.inf]\n            else:\n                value_counts = df[column].value_counts(normalize=True)\n                unique_values = value_counts[value_counts > threshold].index.tolist()\n\n                sorted_values = np.sort(df[column].dropna().unique())\n                cumulative = 0\n                bins = [-np.inf]\n\n                for val in sorted_values:\n                    prop = (df[column] == val).mean()\n                    if prop > threshold:\n                        bins.append(val)\n                    else:\n                        cumulative += prop\n                        if cumulative >= threshold:\n                            bins.append(val)\n                            cumulative = 0\n\n                bins.append(np.inf)\n                bins = sorted(set(bins))\n\n            df['GRP_' + column] = pd.cut(df[column], bins=bins, include_lowest=True, duplicates='drop')\n\n            if missing_mask.any():\n                df['GRP_' + column] = df['GRP_' + column].astype(object)\n                df.loc[missing_mask, 'GRP_' + column] = 'Missing'\n                add_missing = True\n            else:\n                add_missing = False\n\n        # Sắp xếp lại danh mục theo giá trị số học hoặc theo danh sách chữ\n        def extract_lower_bound(x):\n            if isinstance(x, pd.Interval):\n                return x.left  # Lấy giá trị lower bound của khoảng\n            return float('inf')\n\n        if df[column].dtype == 'O':\n            categories = list(df[column].dropna().unique())\n            if add_missing:\n                categories.append('Missing')\n        else:\n            categories = sorted(\n                [cat for cat in df['GRP_' + column].dropna().unique() if isinstance(cat, pd.Interval)],\n                key=extract_lower_bound\n            )\n            if add_missing:\n                categories.append('Missing')  # Thêm Missing vào danh mục cuối cùng nếu có NaN\n\n        df['GRP_' + column] = pd.Categorical(df['GRP_' + column], categories=categories, ordered=True)\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:26:34.164583Z","iopub.execute_input":"2025-04-07T14:26:34.164905Z","iopub.status.idle":"2025-04-07T14:26:34.17853Z","shell.execute_reply.started":"2025-04-07T14:26:34.164878Z","shell.execute_reply":"2025-04-07T14:26:34.177136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Tính WOE, IV:\ndef caculate_WOE_IV(df,feature,target):\n    df = df.groupby(feature,observed=False)[target].agg(['count','sum']).reset_index()\n    df.columns = ['bin','num_of_obs','num_of_event']\n    df['num_of_non_event']= df['num_of_obs'] - df['num_of_event']\n    df['num_of_event'] = df['num_of_event'].astype(float)\n    df['num_of_non_event'] = df['num_of_non_event'].astype(float)\n    df['num_of_non_event_c'] = df['num_of_non_event'].copy()\n    df['num_of_event_c'] = df['num_of_event'].copy()\n    total_non_event = df['num_of_non_event'].sum()\n    total_event = df['num_of_event'].sum()\n    # Nếu event hoặc non_event = 0 thì cộng 0.5 vào cả hai\n    mask = (df['num_of_event'] == 0) | (df['num_of_non_event'] == 0)\n    df.loc[mask, 'num_of_event_c'] += 0.5\n    df.loc[mask, 'num_of_non_event_c'] += 0.5\n    df['prct_non_event']= df['num_of_non_event_c']/total_non_event\n    df['prct_event']= df['num_of_event_c']/total_event\n    df['WOE'] = np.log(df['prct_non_event']/df['prct_event'])\n    df['IV'] = (df['prct_non_event']- df['prct_event']) * df['WOE']\n    IV = df['IV'].sum()\n    df.index = range(1,len(df)+1)\n    df['bin'] = df['bin'].astype(str)\n\n    df = df.drop(columns=['num_of_non_event_c','num_of_event_c'])\n    df['event_rate'] = df['num_of_event']/df['num_of_obs']\n    df['prct_obs'] = df['num_of_obs']/(df['num_of_obs'].sum())\n    return df , IV","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:26:34.179661Z","iopub.execute_input":"2025-04-07T14:26:34.179971Z","iopub.status.idle":"2025-04-07T14:26:34.204536Z","shell.execute_reply.started":"2025-04-07T14:26:34.179947Z","shell.execute_reply":"2025-04-07T14:26:34.203462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def woe_feature_table(data, custom_bins, feature):\n    df_binned = binning_all_variables(data, threshold=0.05, custom_bins=custom_bins, feature=feature)\n    WOE_TABLE, IV = caculate_WOE_IV(df_binned, f'GRP_{feature}', 'target')\n\n    df = WOE_TABLE[['bin', 'num_of_obs', 'prct_obs', 'num_of_non_event', 'num_of_event', 'event_rate', 'WOE', 'IV']]\n    df.columns = ['bin','count','count(%)','non_event','event','event_rate','WOE','IV']\n\n    total_row = pd.DataFrame({\n        'bin': [''],\n        'count': [df['count'].sum()],\n        'count(%)': [df['count(%)'].sum()],\n        'non_event': [df['non_event'].sum()],\n        'event': [df['event'].sum()],\n        'event_rate': [df['event_rate'].sum()],\n        'WOE': [''],\n        'IV': [df['IV'].sum()]\n    })\n\n    df = pd.concat([df, total_row], ignore_index=True)\n    df.index = df.index.astype(str)\n    df.index.values[-1] = \"TOTAL\"\n   \n    return df\ndf2 = woe_feature_table(data_rel, custom_bins = None, feature = 'MONTH')\ndf2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T14:42:18.340314Z","iopub.execute_input":"2025-04-07T14:42:18.340709Z","iopub.status.idle":"2025-04-07T14:42:18.425749Z","shell.execute_reply.started":"2025-04-07T14:42:18.340676Z","shell.execute_reply":"2025-04-07T14:42:18.424657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndf = data_rel\n# Tách tập OOT (tháng 12)\noot_df = df[df['MONTH'] == '201912']\n\n# Tách dữ liệu tháng 11 để chia test/validation\nnov_df = df[df['MONTH'] == '201911']\n\n# Bước 1: Chia 20% cho train, 80% cho test+val\ntrain_part, test_val = train_test_split(\n    nov_df,\n    test_size=0.8,  # Giữ lại 20% cho train\n    random_state=42,\n    stratify=nov_df['target']\n)\n\n# Bước 2: Chia 66% thành test và validation (50-50)\ntest, val= train_test_split(\n    test_val,\n    test_size=0.5, \n    random_state=42,\n    stratify=test_val['target']\n)\n# Tạo tập train (tháng 9 + 10 + phần còn lại của 11)\ntrain_df = pd.concat([\n    df[df['MONTH'].isin(['201909', '201910'])],\n    nov_df.drop(test_part.index).drop(val_part.index)\n])\n\noot_df['MONTH'].value_counts()\nprint(test['target'].value_counts())\nprint(val['target'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T15:04:18.156674Z","iopub.execute_input":"2025-04-07T15:04:18.157053Z","iopub.status.idle":"2025-04-07T15:04:18.225252Z","shell.execute_reply.started":"2025-04-07T15:04:18.157023Z","shell.execute_reply":"2025-04-07T15:04:18.224143Z"}},"outputs":[],"execution_count":null}]}