{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Example Notebook\nTrong bài toán này là bài toán rủi ro tính dụng nên sẽ tập trung và xử lý dữ liệu local thinking về các thông tin của bài toán hơn là chú tâm vào việc xứ lý và tối ưu xử dụng model vậy ta sẽ suy nghĩa và trích lọc các feature để tăng độ chính xác của bài toàn sử dụng thư việc polar. Và sau đây là danh sách các bài bước thực hiện :\n\nIn this notebook you will see how to:\n* Load the data\n* Join tables with Polars - a DataFrame library implemented in Rust language, designed to be blazingy fast and memory efficient.  \n* Create features\n* Train a LightGBM model\n* Create a submission tabl\n\n## Load data","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score \n\ndataPath = \"/kaggle/input/competitions/home-credit-credit-risk-model-stability/\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:28.197358Z","iopub.execute_input":"2026-06-28T09:00:28.197620Z","iopub.status.idle":"2026-06-28T09:00:28.201978Z","shell.execute_reply.started":"2026-06-28T09:00:28.197599Z","shell.execute_reply":"2026-06-28T09:00:28.201380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# CELL NÀY PHẢI CHẠY TRƯỚC TẤT CẢ — định nghĩa lại hàm sạch\n# ============================================================\nimport polars as pl\nimport gc\nimport os\nimport glob\n\ndef set_table_dtypes(df: pl.LazyFrame) -> pl.LazyFrame:\n    schema = df.collect_schema()\n    exprs = []\n\n    for col_name in schema.names():\n        if col_name == \"case_id\":\n            exprs.append(pl.col(col_name))\n\n        elif col_name.startswith(\"num_group\"):\n            exprs.append(pl.col(col_name))\n\n        elif col_name.endswith((\"P\", \"A\")):\n            exprs.append(\n                pl.col(col_name).cast(pl.Float32, strict=False)\n            )\n\n        elif col_name.endswith(\"D\"):\n            exprs.append(pl.col(col_name))\n\n        elif col_name.endswith((\"M\", \"T\")):\n            exprs.append(\n                pl.col(col_name).cast(pl.Utf8, strict=False)\n            )\n\n        elif col_name.endswith(\"L\"):\n            # L có thể là numeric hoặc categorical\n            # Giữ nguyên để tránh lỗi string > numeric\n            exprs.append(pl.col(col_name))\n\n        else:\n            exprs.append(pl.col(col_name))\n\n    return df.with_columns(exprs)\n\ndef downcast_df(df: pl.DataFrame) -> pl.DataFrame:\n    exprs = []\n    for col_name, dtype in zip(df.columns, df.dtypes):\n        if dtype == pl.Float64:\n            exprs.append(pl.col(col_name).cast(pl.Float32))\n        elif dtype == pl.Int64:\n            exprs.append(pl.col(col_name).cast(pl.Int32))\n        elif dtype == pl.Int32:\n            exprs.append(pl.col(col_name).cast(pl.Int16))\n        else:\n            exprs.append(pl.col(col_name))\n    return df.with_columns(exprs)\n\ndef convert_strings(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.copy()\n    for col in df.columns:\n        if df[col].dtype == \"object\" or pd.api.types.is_string_dtype(df[col]):\n            df[col] = df[col].astype(\"category\")\n    return df\n\ndef safe_scan(pattern: str) -> pl.LazyFrame:\n    files = sorted(glob.glob(pattern))\n    if not files:\n        raise FileNotFoundError(\n            f\"Không tìm thấy: {pattern}\\n\"\n            f\"Files có trong thư mục: {sorted(os.listdir(os.path.dirname(pattern)))}\"\n        )\n    print(f\"  ✓ {len(files)} file: {[os.path.basename(f) for f in files]}\")\n    return pl.scan_parquet(files)\n\ndef check_memory(label=\"\"):\n    import psutil\n    ram = psutil.virtual_memory()\n    print(f\"[{label}] RAM: {ram.used/1e9:.1f}GB / {ram.total/1e9:.1f}GB ({ram.percent}%)\")\n\n# Xác nhận hàm đã được load đúng\nimport inspect\nsrc = inspect.getsource(set_table_dtypes)\nassert \"df.columns\" not in src, \"❌ Vẫn còn code cũ!\"\nprint(\"✅ Tất cả hàm đã được định nghĩa đúng, không còn warning\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:29.312564Z","iopub.execute_input":"2026-06-28T09:00:29.313328Z","iopub.status.idle":"2026-06-28T09:00:29.326826Z","shell.execute_reply.started":"2026-06-28T09:00:29.313267Z","shell.execute_reply":"2026-06-28T09:00:29.326041Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Đọc bản dữ liệu\n\nTrong phần này load toàn bộ dữ liệu lên trên máy","metadata":{}},{"cell_type":"code","source":"check_memory(\"START\")\n\ntrain_basetable = pl.scan_parquet(dataPath + \"parquet_files/train/train_base.parquet\")\ntrain_static = pl.scan_parquet(dataPath + \"parquet_files/train/train_static_0_*.parquet\").pipe(set_table_dtypes)\n\ntrain_static_cb = pl.scan_parquet(dataPath + \"parquet_files/train/train_static_cb_0.parquet\").pipe(set_table_dtypes)\ntrain_person_1 = pl.scan_parquet(dataPath + \"parquet_files/train/train_person_1.parquet\").pipe(set_table_dtypes) \ntrain_credit_bureau_a_2 = pl.scan_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_2_*.parquet\").pipe(set_table_dtypes) \ntrain_credit_bureau_b_1 = pl.scan_parquet(dataPath + \"parquet_files/train/train_credit_bureau_b_1.parquet\").pipe(set_table_dtypes) \ntrain_credit_bureau_b_2 = pl.scan_parquet(dataPath + \"parquet_files/train/train_credit_bureau_b_2.parquet\").pipe(set_table_dtypes) \ntrain_credit_bureau_a_1 = pl.scan_parquet(dataPath + \"parquet_files/train/train_credit_bureau_a_1_*.parquet\").pipe(set_table_dtypes)\ntrain_tax_registry_a_1  = pl.scan_parquet(dataPath + \"parquet_files/train/train_tax_registry_a_1.parquet\").pipe(set_table_dtypes)\ntrain_applprev_1 = pl.scan_parquet(dataPath + \"parquet_files/train/train_applprev_1_*.parquet\").pipe(set_table_dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:32.572144Z","iopub.execute_input":"2026-06-28T09:00:32.572473Z","iopub.status.idle":"2026-06-28T09:00:32.698647Z","shell.execute_reply.started":"2026-06-28T09:00:32.572447Z","shell.execute_reply":"2026-06-28T09:00:32.697955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_basetable = pl.scan_parquet(dataPath + \"parquet_files/test/test_base.parquet\")\ntest_static = pl.scan_parquet(dataPath + \"parquet_files/test/test_static_0_*.parquet\").pipe(set_table_dtypes)\ntest_static_cb = pl.scan_parquet(dataPath + \"parquet_files/test/test_static_cb_0.parquet\").pipe(set_table_dtypes)\ntest_person_1 = pl.scan_parquet(dataPath + \"parquet_files/test/test_person_1.parquet\").pipe(set_table_dtypes) \ntest_credit_bureau_a_2 = pl.scan_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_2_*.parquet\").pipe(set_table_dtypes)\ntest_credit_bureau_b_2 = pl.scan_parquet(dataPath + \"parquet_files/test/test_credit_bureau_b_2.parquet\").pipe(set_table_dtypes)\ntest_credit_bureau_b_1 = pl.scan_parquet(dataPath + \"parquet_files/test/test_credit_bureau_b_1.parquet\").pipe(set_table_dtypes)\ntest_credit_bureau_a_1 = pl.scan_parquet(dataPath + \"parquet_files/test/test_credit_bureau_a_1_*.parquet\").pipe(set_table_dtypes)\ntest_tax_registry_a_1 = pl.scan_parquet(dataPath + \"parquet_files/test/test_tax_registry_a_1.parquet\").pipe(set_table_dtypes) \ntest_applprev_1 = pl.scan_parquet(dataPath + \"parquet_files/test/test_applprev_1_*.parquet\").pipe(set_table_dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:34.005843Z","iopub.execute_input":"2026-06-28T09:00:34.006568Z","iopub.status.idle":"2026-06-28T09:00:34.113077Z","shell.execute_reply.started":"2026-06-28T09:00:34.006533Z","shell.execute_reply":"2026-06-28T09:00:34.112533Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature engineering\nTrong phần này mình sẽ thực hiện trích xuất ra các đặc trưng dựa vào dữ liệu mà ta có thể thấy bằng việc tổng hợp và xử lý lại dữ liệu:\n1. Ta sẽ xem thu nhập tài chính lớn nhất mà một khách hàng làm nghề tự do có được. Vì việc làm tự do với mức thu nhập không cố định có thể tạo ra các rủi ro về mặt tài chính từ đó cần được quan tâm tới.\n2. Ta sẽ xét về mặt nhà ở của người chủ chính đứng ra làm hợp đồng cho thuê. Đánh giá sự ổn định của một khách hàng nếu họ không có nhà hay nhà thuê rủi ro họ bốc hơi là có khả năng.\n3. Xem xét xem mức nợ quá hạn cao nhất mà người đó chi trả một người từng nợ quá hạn thấp rủi ro tài chính sẽ thấp hơn. Việc khách hàng trễ hạn chi trả 1 tháng > 31 ngày không thể nào là quên mà là chứng tỏ dòng tiền của họ bị đứt gãy và họ hiện tại không có khả năng chi trả rủi ro dồn nợ là rất cao.\n4. cũng cần xem thêm tỷ lệ nợ trên thu nhập\n5. Xu hướng xấu đi của hành vi thanh toán xem xem số ngày quá hạn của tháng này thấp hơn tháng trước không ? Xem xem mà max DPD gần đây của khách hàng hay là tỷ lệ trung bình phần trăm các khoản trả góp được thanh toán từ 1 ngày trở lên sau ngày đáo hạn.\n6. Xem xét về mức độ sử dụng tính dụng\n\n#### Setup bộ công cụ hỗ trợ tìm kiếm","metadata":{}},{"cell_type":"code","source":"import polars as pl\n\ndef has_column(df: pl.DataFrame, column_name: str) -> bool:\n    \"\"\"\n    Kiểm tra một tên cột có tồn tại trong DataFrame hay không.\n    Trả về True nếu có, False nếu không.\n    \"\"\"\n    # Cách nhanh nhất trong Polars là kiểm tra trong danh sách df.columns hoặc df.schema\n    if column_name in df.columns:\n        print(f\"✅ Cột '{column_name}' CÓ tồn tại trong bảng.\")\n        return True\n    else:\n        print(f\"❌ Cột '{column_name}' KHÔNG tồn tại trong bảng.\")\n        return False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:36.877092Z","iopub.execute_input":"2026-06-28T09:00:36.877559Z","iopub.status.idle":"2026-06-28T09:00:36.882475Z","shell.execute_reply.started":"2026-06-28T09:00:36.877527Z","shell.execute_reply":"2026-06-28T09:00:36.881459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport os\nimport polars as pl\n\n\ndef find_column_in_folder(target_column: str, folder_path: str) -> list:\n    \"\"\"Hàm quét toàn bộ file .csv trong thư mục để tìm tên cột chỉ định.\n\n    Chỉ đọc schema đầu file nên tốc độ cực nhanh và siêu tiết kiệm RAM.\n    \"\"\"\n    # Lấy đường dẫn tuyệt đối của tất cả các file .csv\n    search_pattern = os.path.join(folder_path, \"*.csv\")\n    csv_files = glob.glob(search_pattern)\n\n    matched_files = []\n\n    print(f\"🕵️ Đang tiến hành quét {len(csv_files)} file .csv...\")\n\n    for file_path in csv_files:\n        try:\n            # scan_csv chỉ đọc cấu trúc cột (schema) chứ không load dữ liệu dòng\n            schema = pl.scan_csv(file_path).schema\n\n            if target_column in schema:\n                # Tách lấy tên file để hiển thị cho gọn thay vì lấy đường dẫn dài\n                file_name = os.path.basename(file_path)\n                matched_files.append(file_name)\n\n        except Exception as e:\n            # Bỏ qua và báo lỗi nếu gặp file rỗng hoặc bị hỏng cấu trúc\n            print(f\"⚠️ Lỗi khi đọc cấu trúc file {os.path.basename(file_path)}: {e}\")\n\n    # In kết quả tổng hợp ra màn hình\n    if matched_files:\n        print(\n            f\"\\n✅ Tìm thấy cột '{target_column}' xuất hiện trong {len(matched_files)} file sau:\"\n        )\n        for f in sorted(matched_files):\n            print(f\"   🔹 {f}\")\n    else:\n        print(\n            f\"\\n❌ Không tìm thấy cột '{target_column}' trong bất kỳ file .csv nào tại thư mục này.\"\n        )\n\n    return matched_files\n\n\n# ---------------------------------------------------------\n# CẤU HÌNH ĐƯỜNG DẪN VÀ CHẠY THỬ NGHIỆM\n# ---------------------------------------------------------\nKAGGLE_TRAIN_DIR = \"/kaggle/input/competitions/home-credit-credit-risk-model-stability/csv_files/train\"\n# Ví dụ 2: Tìm cột tổng nợ quá hạn\n# found_files = find_column_in_folder(\"totaldebtoverduevalue_178A\", KAGGLE_TRAIN_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:37.910631Z","iopub.execute_input":"2026-06-28T09:00:37.911426Z","iopub.status.idle":"2026-06-28T09:00:37.918065Z","shell.execute_reply.started":"2026-06-28T09:00:37.911397Z","shell.execute_reply":"2026-06-28T09:00:37.917365Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Thực hiện thiết kế data","metadata":{}},{"cell_type":"markdown","source":"#### Thực hiện các rủ ro tài chính cơ bản từ mẫu","metadata":{}},{"cell_type":"code","source":"# We need to use aggregation functions in tables with depth > 1, so tables that contain num_group1 column or \n# also num_group2 column.\ntrain_person_1_feats_1 = train_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\n# Here num_group1=0 has special meaning, it is the person who applied for the loan.\ntrain_person_1_feats_2 = train_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\n# Here we have num_goup1 and num_group2, so we need to aggregate again.\ntrain_credit_bureau_b_2_feats = train_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:41.013470Z","iopub.execute_input":"2026-06-28T09:00:41.013722Z","iopub.status.idle":"2026-06-28T09:00:41.019938Z","shell.execute_reply.started":"2026-06-28T09:00:41.013702Z","shell.execute_reply":"2026-06-28T09:00:41.019213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_person_1_feats_1 = test_person_1.group_by(\"case_id\").agg(\n    pl.col(\"mainoccupationinc_384A\").max().alias(\"mainoccupationinc_384A_max\"),\n    (pl.col(\"incometype_1044T\") == \"SELFEMPLOYED\").max().alias(\"mainoccupationinc_384A_any_selfemployed\")\n)\n\ntest_person_1_feats_2 = test_person_1.select([\"case_id\", \"num_group1\", \"housetype_905L\"]).filter(\n    pl.col(\"num_group1\") == 0\n).drop(\"num_group1\").rename({\"housetype_905L\": \"person_housetype\"})\n\ntest_credit_bureau_b_2_feats = test_credit_bureau_b_2.group_by(\"case_id\").agg(\n    pl.col(\"pmts_pmtsoverdue_635A\").max().alias(\"pmts_pmtsoverdue_635A_max\"),\n    (pl.col(\"pmts_dpdvalue_108P\") > 31).max().alias(\"pmts_dpdvalue_108P_over31\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:42.468729Z","iopub.execute_input":"2026-06-28T09:00:42.469013Z","iopub.status.idle":"2026-06-28T09:00:42.475181Z","shell.execute_reply.started":"2026-06-28T09:00:42.468989Z","shell.execute_reply":"2026-06-28T09:00:42.474322Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Debt Burden & Affordability (Gánh nặng nợ)\n\n\ntrong phần này sẽ xử lý gánh nặng nợ  ta sẽ xử lý để lấy ra thu nhập của người vay chính và nợ quá hạn lớn nhât của người đó sau đó xử lý:\n\n1. Tính tỷ lệ dựa trên tổng nợ/ thu nhập chính của khách hàng để đánh giá được sự chệch lệch giữa thu nhập và nợ có đồng đều hay không ?\n2. Xem xét về khả năng trả nợ hàng tháng.\n3. Tính tỷ lệ tổng nợ đang quá hạn và tổng dư nợ để biết được mức độ nghiêm trọng của sự cố đỗ vỡ tài chính ví dụ:\nA: tổng dư nợ là 2 tỷ đồng nợ quá hạn là 10 thì rủi ro sẽ thấp hơn người tổng nợ 2 tỷ mà nợ quá hạn là 200tr","metadata":{}},{"cell_type":"code","source":"# Debt-to-Income Ratio (DTI) - Chỉ số vàng trong tín dụng\n# 1.1  lấy thu nhập của NGƯỜI VAY CHÍNH (num_group1 = 0)\nperson_flat = train_person_1.filter(pl.col(\"num_group1\") == 0).select([\n    \"case_id\", \n    \"mainoccupationinc_384A\"\n])\n\n# 1.2  nhóm lấy Tổng (hoặc Max) nợ quá hạn\n# (Giả định cột này nằm ở bảng credit_bureau_a_1 như code bạn viết)\ncb_a_1_flat = train_credit_bureau_a_1.group_by(\"case_id\").agg([\n    pl.col(\"totaldebtoverduevalue_178A\").max().alias(\"totaldebtoverduevalue_178A_max\"),\n    pl.col(\"totaloutstanddebtvalue_39A\").max().alias(\"totaloutstanddebtvalue_39A_max\"),\n])\n\nmaster_data = train_static.join(\n    person_flat, how=\"left\", on=\"case_id\"\n).join(\n    cb_a_1_flat, how=\"left\", on=\"case_id\"\n)\n\ntrain_static_feats = master_data.with_columns([\n    (pl.col(\"totaldebt_9A\") / (pl.col(\"mainoccupationinc_384A\") + 1))\n        .alias(\"debt_to_income_ratio\"),\n    \n    (pl.col(\"annuity_780A\") / (pl.col(\"mainoccupationinc_384A\") + 1))\n        .alias(\"annuity_to_income_ratio\"),\n    \n    (pl.col(\"totaldebtoverduevalue_178A_max\") / (pl.col(\"totaloutstanddebtvalue_39A_max\") + 1))\n        .alias(\"overdue_to_total_debt_ratio\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:43.895427Z","iopub.execute_input":"2026-06-28T09:00:43.896080Z","iopub.status.idle":"2026-06-28T09:00:43.902950Z","shell.execute_reply.started":"2026-06-28T09:00:43.896040Z","shell.execute_reply":"2026-06-28T09:00:43.901745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Debt-to-Income Ratio (DTI) - Chỉ số vàng trong tín dụng\n# 1.1  lấy thu nhập của NGƯỜI VAY CHÍNH (num_group1 = 0)\nperson_flat = test_person_1.filter(pl.col(\"num_group1\") == 0).select([\n    \"case_id\", \n    \"mainoccupationinc_384A\"\n])\n\n# 1.2  nhóm lấy Tổng (hoặc Max) nợ quá hạn\n# (Giả định cột này nằm ở bảng credit_bureau_a_1 như code bạn viết)\ncb_a_1_flat = test_credit_bureau_a_1.group_by(\"case_id\").agg([\n    pl.col(\"totaldebtoverduevalue_178A\").max().alias(\"totaldebtoverduevalue_178A_max\"),\n    pl.col(\"totaloutstanddebtvalue_39A\").max().alias(\"totaloutstanddebtvalue_39A_max\"),\n])\n\nmaster_data = test_static.join(\n    person_flat, how=\"left\", on=\"case_id\"\n).join(\n    cb_a_1_flat, how=\"left\", on=\"case_id\"\n)\n\ntest_static_feats = master_data.with_columns([\n    (pl.col(\"totaldebt_9A\") / (pl.col(\"mainoccupationinc_384A\") + 1))\n        .alias(\"debt_to_income_ratio\"),\n    \n    (pl.col(\"annuity_780A\") / (pl.col(\"mainoccupationinc_384A\") + 1))\n        .alias(\"annuity_to_income_ratio\"),\n    \n    (pl.col(\"totaldebtoverduevalue_178A_max\") / (pl.col(\"totaloutstanddebtvalue_39A_max\") + 1))\n        .alias(\"overdue_to_total_debt_ratio\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:45.098857Z","iopub.execute_input":"2026-06-28T09:00:45.099788Z","iopub.status.idle":"2026-06-28T09:00:45.106463Z","shell.execute_reply.started":"2026-06-28T09:00:45.099742Z","shell.execute_reply":"2026-06-28T09:00:45.105632Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Payment Behavior Deterioration (Xu hướng xấu đi của hành vi thanh toán)\n\n\n- ta sẽ xem xét xu hướng hành vi nếu mà số ngày quá hạn trung bình của 3 tháng gần nhất lớn hơn nhiều lần với số ngày quá hạn trong 24 tháng vừa qua điều này chứng tỏ tình hình tài chính kinh tế ngày càng tệ đi.\n- Xem xét Trung bình của các đỉnh điểm trễ hạn trong 9 tháng qua. Nếu chỉ số này cao chứng tỏ khách hàng này có thói quen chiến dụng vốn và dòng tiền đang bất ổn. Dù chưa chạm mức nợ xấu nhưng hành vì này kéo dài rồng rả 3 quý cũng là 1 rủi ro.\n- Xem xét về thời gian trong khoảng thời gian 2 năm qua một người đã ao lần chuyển tiền trả nợ","metadata":{}},{"cell_type":"code","source":"train_credit_bureau_trend = train_static.group_by(\"case_id\").agg([\n    (pl.col(\"avgdbddpdlast3m_4187120P\") - pl.col(\"avgdbddpdlast24m_3658932P\"))\n        .mean().alias(\"dpd_trend_3m_vs_24m\"),\n\n    pl.col(\"avgmaxdpdlast9m_3716943P\").max().alias(\"max_dpd_9m\"),\n\n    (pl.col(\"cntpmts24_3658933L\").cast(pl.Float32) / 24)\n        .mean().alias(\"payment_frequency_ratio\"),\n\n    pl.col(\"pctinstlsallpaidlate1d_3546856L\")\n        .cast(pl.Float32)\n        .mean().alias(\"late_payment_pct\"),\n\n    pl.col(\"pctinstlsallpaidlat10d_839L\")\n        .cast(pl.Float32)\n        .mean().alias(\"very_late_payment_pct\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:47.444275Z","iopub.execute_input":"2026-06-28T09:00:47.444804Z","iopub.status.idle":"2026-06-28T09:00:47.449908Z","shell.execute_reply.started":"2026-06-28T09:00:47.444774Z","shell.execute_reply":"2026-06-28T09:00:47.449385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_credit_bureau_trend = test_static.group_by(\"case_id\").agg([\n    (pl.col(\"avgdbddpdlast3m_4187120P\") - pl.col(\"avgdbddpdlast24m_3658932P\"))\n        .mean().alias(\"dpd_trend_3m_vs_24m\"),\n\n    pl.col(\"avgmaxdpdlast9m_3716943P\").max().alias(\"max_dpd_9m\"),\n\n    (pl.col(\"cntpmts24_3658933L\").cast(pl.Float32) / 24)\n        .mean().alias(\"payment_frequency_ratio\"),\n\n    pl.col(\"pctinstlsallpaidlate1d_3546856L\")\n        .cast(pl.Float32)\n        .mean().alias(\"late_payment_pct\"),\n\n    pl.col(\"pctinstlsallpaidlat10d_839L\")\n        .cast(pl.Float32)\n        .mean().alias(\"very_late_payment_pct\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:49.713362Z","iopub.execute_input":"2026-06-28T09:00:49.714145Z","iopub.status.idle":"2026-06-28T09:00:49.719048Z","shell.execute_reply.started":"2026-06-28T09:00:49.714106Z","shell.execute_reply":"2026-06-28T09:00:49.718347Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Credit Utilization & Exposure (Mức độ sử dụng tín dụng)","metadata":{}},{"cell_type":"code","source":"train_credacc = train_applprev_1.group_by(\"case_id\").agg([\n    pl.col(\"credacc_actualbalance_314A\").mean().alias(\"credacc_actualbalance_314A_mean\"),\n    pl.col(\"credacc_credlmt_575A\").max().alias(\"credacc_credlmt_575A_mean\"),\n])\n\n\ntrain_credit_bureau_b_1_for_exposure = train_credit_bureau_b_1.group_by(\"case_id\").agg([\n    pl.col(\"residualamount_3940956A\").mean().alias(\"residualamount_3940956A_mean\"),\n    pl.col(\"credlmt_1052A\").max().alias(\"credlmt_1052A_mean\"),\n])\n\n\nmater_data = train_credit_bureau_a_1.join(\n    train_credacc, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_1_for_exposure, how=\"left\", on=\"case_id\"\n)\n\n\ntrain_credit_exposure = mater_data.group_by(\"case_id\").agg([\n    # Utilization ratio: Số dư thực tế / Hạn mức\n    (pl.col(\"credacc_actualbalance_314A_mean\") / (pl.col(\"credacc_credlmt_575A_mean\") + 1))\n        .mean().alias(\"credit_utilization_ratio\"),\n    \n    # Số hợp đồng tín dụng đang active\n    (pl.col(\"numberofcontrsvalue_258L\")).max().alias(\"active_contracts_count\"),\n    \n    # Tổng exposure so với thu nhập\n    pl.col(\"totalamount_996A\").max().alias(\"total_active_credit_exposure\"),\n    \n    # Tỷ lệ residual còn lại (nợ gốc còn lại)\n    (pl.col(\"residualamount_3940956A_mean\") / (pl.col(\"credlmt_1052A_mean\") + 1))\n        .mean().alias(\"residual_ratio\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:51.044160Z","iopub.execute_input":"2026-06-28T09:00:51.044969Z","iopub.status.idle":"2026-06-28T09:00:51.051639Z","shell.execute_reply.started":"2026-06-28T09:00:51.044937Z","shell.execute_reply":"2026-06-28T09:00:51.050654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_credacc = test_applprev_1.group_by(\"case_id\").agg([\n    pl.col(\"credacc_actualbalance_314A\").mean().alias(\"credacc_actualbalance_314A_mean\"),\n    pl.col(\"credacc_credlmt_575A\").max().alias(\"credacc_credlmt_575A_mean\"),\n])\n\n\ntest_credit_bureau_b_1_for_exposure = test_credit_bureau_b_1.group_by(\"case_id\").agg([\n    pl.col(\"residualamount_3940956A\").mean().alias(\"residualamount_3940956A_mean\"),\n    pl.col(\"credlmt_1052A\").max().alias(\"credlmt_1052A_mean\"),\n])\n\n\nmater_data = test_credit_bureau_a_1.join(\n    test_credacc, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_1_for_exposure, how=\"left\", on=\"case_id\"\n)\n\n\ntest_credit_exposure = mater_data.group_by(\"case_id\").agg([\n    # Utilization ratio: Số dư thực tế / Hạn mức\n    (pl.col(\"credacc_actualbalance_314A_mean\") / (pl.col(\"credacc_credlmt_575A_mean\") + 1))\n        .mean().alias(\"credit_utilization_ratio\"),\n    \n    # Số hợp đồng tín dụng đang active\n    (pl.col(\"numberofcontrsvalue_258L\")).max().alias(\"active_contracts_count\"),\n    \n    # Tổng exposure so với thu nhập\n    pl.col(\"totalamount_996A\").max().alias(\"total_active_credit_exposure\"),\n    \n    # Tỷ lệ residual còn lại (nợ gốc còn lại)\n    (pl.col(\"residualamount_3940956A_mean\") / (pl.col(\"credlmt_1052A_mean\") + 1))\n        .mean().alias(\"residual_ratio\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:51.294586Z","iopub.execute_input":"2026-06-28T09:00:51.295363Z","iopub.status.idle":"2026-06-28T09:00:51.301761Z","shell.execute_reply.started":"2026-06-28T09:00:51.295331Z","shell.execute_reply":"2026-06-28T09:00:51.300733Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Application Fraud & Behavioral Signals (Tín hiệu gian lận)","metadata":{}},{"cell_type":"code","source":"train_fraud_signals = train_static.select([\n    \"case_id\",\n\n    pl.col(\"clientscnt_304L\").cast(pl.Float32, strict=False).alias(\"same_phone_clients\"),\n    pl.col(\"clientscnt_136L\").cast(pl.Float32, strict=False).alias(\"same_email_clients\"),\n    pl.col(\"applicationscnt_1086L\").cast(pl.Float32, strict=False).alias(\"same_phone_apps\"),\n    pl.col(\"applications30d_658L\").cast(pl.Float32, strict=False).alias(\"apps_last_30d\"),\n    pl.col(\"applicationscnt_464L\").cast(pl.Float32, strict=False).alias(\"same_employer_apps\"),\n]).with_columns([\n    (\n        (pl.col(\"same_phone_clients\") > 2).cast(pl.Int32)\n        + (pl.col(\"same_email_clients\") > 2).cast(pl.Int32)\n        + (pl.col(\"apps_last_30d\") > 3).cast(pl.Int32)\n    ).alias(\"fraud_signal_score\")\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:54.054232Z","iopub.execute_input":"2026-06-28T09:00:54.054857Z","iopub.status.idle":"2026-06-28T09:00:54.060448Z","shell.execute_reply.started":"2026-06-28T09:00:54.054825Z","shell.execute_reply":"2026-06-28T09:00:54.059584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_fraud_signals = test_static.select([\n    \"case_id\",\n\n    pl.col(\"clientscnt_304L\").cast(pl.Float32, strict=False).alias(\"same_phone_clients\"),\n    pl.col(\"clientscnt_136L\").cast(pl.Float32, strict=False).alias(\"same_email_clients\"),\n    pl.col(\"applicationscnt_1086L\").cast(pl.Float32, strict=False).alias(\"same_phone_apps\"),\n    pl.col(\"applications30d_658L\").cast(pl.Float32, strict=False).alias(\"apps_last_30d\"),\n    pl.col(\"applicationscnt_464L\").cast(pl.Float32, strict=False).alias(\"same_employer_apps\"),\n]).with_columns([\n    (\n        (pl.col(\"same_phone_clients\") > 2).cast(pl.Int32)\n        + (pl.col(\"same_email_clients\") > 2).cast(pl.Int32)\n        + (pl.col(\"apps_last_30d\") > 3).cast(pl.Int32)\n    ).alias(\"fraud_signal_score\")\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:56.314914Z","iopub.execute_input":"2026-06-28T09:00:56.315189Z","iopub.status.idle":"2026-06-28T09:00:56.320989Z","shell.execute_reply.started":"2026-06-28T09:00:56.315165Z","shell.execute_reply":"2026-06-28T09:00:56.320238Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"####  Employment & Income Stability (Mức độ ổn định thu nhập)","metadata":{}},{"cell_type":"code","source":"# Từ tax registry — thu nhập có đều không?\n\n\n\ntrain_tax_stability = train_static_cb.group_by(\"case_id\").agg([\n    # Số lần có tax deduction (proxy cho tháng đi làm)\n    pl.col(\"pmtcount_693L\").max().alias(\"tax_payment_count\"),\n    \n    # Biến động thu nhập (std / mean)\n    (pl.col(\"pmtaverage_3A\").std() / (pl.col(\"pmtaverage_3A\").mean() + 1))\n        .alias(\"income_volatility\"),  # Cao = thu nhập không ổn định\n    \n    # Tổng thu nhập đã khai thuế\n    pl.col(\"pmtssum_45A\").sum().alias(\"total_declared_income\"),\n])\n\n# So sánh thu nhập khai báo vs thu nhập trong đơn vay\n# Nếu chênh lệch lớn = rủi ro gian lận hoặc over-leverage","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:57.663595Z","iopub.execute_input":"2026-06-28T09:00:57.663860Z","iopub.status.idle":"2026-06-28T09:00:57.669035Z","shell.execute_reply.started":"2026-06-28T09:00:57.663837Z","shell.execute_reply":"2026-06-28T09:00:57.668192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Từ tax registry — thu nhập có đều không?\ntest_tax_stability = test_static_cb.group_by(\"case_id\").agg([\n    # Số lần có tax deduction (proxy cho tháng đi làm)\n    pl.col(\"pmtcount_693L\").max().alias(\"tax_payment_count\"),\n    \n    # Biến động thu nhập (std / mean)\n    (pl.col(\"pmtaverage_3A\").std() / (pl.col(\"pmtaverage_3A\").mean() + 1))\n        .alias(\"income_volatility\"),  # Cao = thu nhập không ổn định\n    \n    # Tổng thu nhập đã khai thuế\n    pl.col(\"pmtssum_45A\").sum().alias(\"total_declared_income\"),\n])\n\n# So sánh thu nhập khai báo vs thu nhập trong đơn vay\n# Nếu chênh lệch lớn = rủi ro gian lận hoặc over-leverage","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:57.860617Z","iopub.execute_input":"2026-06-28T09:00:57.860837Z","iopub.status.idle":"2026-06-28T09:00:57.865691Z","shell.execute_reply.started":"2026-06-28T09:00:57.860817Z","shell.execute_reply":"2026-06-28T09:00:57.865007Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Recency-Frequency-Monetary (RFM) trên lịch sử tín dụng","metadata":{}},{"cell_type":"code","source":"train_credit_bureau_b_2_enhanced = train_credit_bureau_b_2.group_by(\"case_id\").agg([\n    # Frequency: số lần quá hạn\n    pl.col(\"pmts_dpdvalue_108P\")\n        .filter(pl.col(\"pmts_dpdvalue_108P\") > 0)\n        .count()\n        .alias(\"overdue_payment_count\"),\n\n    # Monetary: tổng số tiền quá hạn\n    pl.col(\"pmts_pmtsoverdue_635A\")\n        .sum()\n        .alias(\"total_overdue_amount\"),\n\n    # Severity: DPD > 31 ngày\n    (pl.col(\"pmts_dpdvalue_108P\") > 31)\n        .sum()\n        .alias(\"severe_dpd_count\"),\n\n    # DPD > 90 ngày\n    (pl.col(\"pmts_dpdvalue_108P\") > 90)\n        .max()\n        .cast(pl.Int8)\n        .alias(\"has_90dpd_ever\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:00:59.610373Z","iopub.execute_input":"2026-06-28T09:00:59.610640Z","iopub.status.idle":"2026-06-28T09:00:59.615912Z","shell.execute_reply.started":"2026-06-28T09:00:59.610618Z","shell.execute_reply":"2026-06-28T09:00:59.615234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_credit_bureau_b_2_enhanced = test_credit_bureau_b_2.group_by(\"case_id\").agg([\n    # Frequency: số lần quá hạn\n    pl.col(\"pmts_dpdvalue_108P\")\n        .filter(pl.col(\"pmts_dpdvalue_108P\") > 0)\n        .count()\n        .alias(\"overdue_payment_count\"),\n\n    # Monetary: tổng số tiền quá hạn\n    pl.col(\"pmts_pmtsoverdue_635A\")\n        .sum()\n        .alias(\"total_overdue_amount\"),\n\n    # Severity: DPD > 31 ngày\n    (pl.col(\"pmts_dpdvalue_108P\") > 31)\n        .sum()\n        .alias(\"severe_dpd_count\"),\n\n    # DPD > 90 ngày\n    (pl.col(\"pmts_dpdvalue_108P\") > 90)\n        .max()\n        .cast(pl.Int8)\n        .alias(\"has_90dpd_ever\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:00.629558Z","iopub.execute_input":"2026-06-28T09:01:00.629823Z","iopub.status.idle":"2026-06-28T09:01:00.635016Z","shell.execute_reply.started":"2026-06-28T09:01:00.629801Z","shell.execute_reply":"2026-06-28T09:01:00.634121Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"####  Collateral Quality & Loan-to-Value (Chất lượng tài sản đảm bảo)","metadata":{}},{"cell_type":"code","source":"collateral_flat = train_credit_bureau_a_2.group_by(\"case_id\").agg([\n    pl.col(\"collater_valueofguarantee_1124L\")\n        .cast(pl.Float32)\n        .max()\n        .alias(\"max_collateral_value\"),\n\n    pl.col(\"collater_valueofguarantee_1124L\")\n        .cast(pl.Float32)\n        .sum()\n        .alias(\"total_collateral_value\"),\n\n    (pl.col(\"collater_valueofguarantee_1124L\").cast(pl.Float32) > 0)\n        .max()\n        .cast(pl.Int8)\n        .alias(\"has_collateral\"),\n])\n\ntrain_collateral = (\n    collateral_flat\n    .join(\n        train_static.select([\"case_id\", \"credamount_770A\"]),\n        how=\"left\",\n        on=\"case_id\"\n    )\n    .with_columns([\n        (pl.col(\"credamount_770A\") / (pl.col(\"max_collateral_value\") + 1))\n            .alias(\"loan_to_value_ratio\")\n    ])\n    .select([\n        \"case_id\",\n        \"max_collateral_value\",\n        \"total_collateral_value\",\n        \"has_collateral\",\n        \"loan_to_value_ratio\",\n    ])\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:02.772594Z","iopub.execute_input":"2026-06-28T09:01:02.773304Z","iopub.status.idle":"2026-06-28T09:01:02.779141Z","shell.execute_reply.started":"2026-06-28T09:01:02.773255Z","shell.execute_reply":"2026-06-28T09:01:02.778414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"collateral_flat = test_credit_bureau_a_2.group_by(\"case_id\").agg([\n    pl.col(\"collater_valueofguarantee_1124L\")\n        .cast(pl.Float32)\n        .max()\n        .alias(\"max_collateral_value\"),\n\n    pl.col(\"collater_valueofguarantee_1124L\")\n        .cast(pl.Float32)\n        .sum()\n        .alias(\"total_collateral_value\"),\n\n    (pl.col(\"collater_valueofguarantee_1124L\").cast(pl.Float32) > 0)\n        .max()\n        .cast(pl.Int8)\n        .alias(\"has_collateral\"),\n])\n\ntest_collateral = (\n    collateral_flat\n    .join(\n        train_static.select([\"case_id\", \"credamount_770A\"]),\n        how=\"left\",\n        on=\"case_id\"\n    )\n    .with_columns([\n        (pl.col(\"credamount_770A\") / (pl.col(\"max_collateral_value\") + 1))\n            .alias(\"loan_to_value_ratio\")\n    ])\n    .select([\n        \"case_id\",\n        \"max_collateral_value\",\n        \"total_collateral_value\",\n        \"has_collateral\",\n        \"loan_to_value_ratio\",\n    ])\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:04.382669Z","iopub.execute_input":"2026-06-28T09:01:04.383385Z","iopub.status.idle":"2026-06-28T09:01:04.388691Z","shell.execute_reply.started":"2026-06-28T09:01:04.383352Z","shell.execute_reply":"2026-06-28T09:01:04.387854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Temporal Risk Features (Rủi ro theo thời gian)","metadata":{}},{"cell_type":"code","source":"train_basetable = train_basetable.with_columns(\n    pl.col(\"date_decision\").cast(pl.Date)\n).with_columns([\n    # Month\n    pl.col(\"date_decision\").dt.month().alias(\"application_month\"),\n\n    # Quarter\n    pl.col(\"date_decision\").dt.quarter().alias(\"application_quarter\"),\n\n    # Week of year\n    pl.col(\"date_decision\").dt.week().alias(\"application_week\"),\n\n    # Day of month\n    pl.col(\"date_decision\").dt.day().alias(\"application_day\"),\n\n    # Weekday (Mon=1 ... Sun=7)\n    pl.col(\"date_decision\").dt.weekday().alias(\"application_weekday\"),\n\n    # Weekend\n    (pl.col(\"date_decision\").dt.weekday() >= 6)\n        .cast(pl.Int8)\n        .alias(\"is_weekend\"),\n\n    # End of month\n    (pl.col(\"date_decision\").dt.day() >= 25)\n        .cast(pl.Int8)\n        .alias(\"is_end_of_month\"),\n\n    # Beginning of month\n    (pl.col(\"date_decision\").dt.day() <= 5)\n        .cast(pl.Int8)\n        .alias(\"is_begin_month\"),\n\n    # Middle of month\n    (\n        (pl.col(\"date_decision\").dt.day() >= 10) &\n        (pl.col(\"date_decision\").dt.day() <= 20)\n    ).cast(pl.Int8).alias(\"is_mid_month\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:06.156624Z","iopub.execute_input":"2026-06-28T09:01:06.157241Z","iopub.status.idle":"2026-06-28T09:01:06.163393Z","shell.execute_reply.started":"2026-06-28T09:01:06.157211Z","shell.execute_reply":"2026-06-28T09:01:06.162537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_basetable = test_basetable.with_columns(\n    pl.col(\"date_decision\").cast(pl.Date)\n).with_columns([\n    # Month\n    pl.col(\"date_decision\").dt.month().alias(\"application_month\"),\n\n    # Quarter\n    pl.col(\"date_decision\").dt.quarter().alias(\"application_quarter\"),\n\n    # Week of year\n    pl.col(\"date_decision\").dt.week().alias(\"application_week\"),\n\n    # Day of month\n    pl.col(\"date_decision\").dt.day().alias(\"application_day\"),\n\n    # Weekday (Mon=1 ... Sun=7)\n    pl.col(\"date_decision\").dt.weekday().alias(\"application_weekday\"),\n\n    # Weekend\n    (pl.col(\"date_decision\").dt.weekday() >= 6)\n        .cast(pl.Int8)\n        .alias(\"is_weekend\"),\n\n    # End of month\n    (pl.col(\"date_decision\").dt.day() >= 25)\n        .cast(pl.Int8)\n        .alias(\"is_end_of_month\"),\n\n    # Beginning of month\n    (pl.col(\"date_decision\").dt.day() <= 5)\n        .cast(pl.Int8)\n        .alias(\"is_begin_month\"),\n\n    # Middle of month\n    (\n        (pl.col(\"date_decision\").dt.day() >= 10) &\n        (pl.col(\"date_decision\").dt.day() <= 20)\n    ).cast(pl.Int8).alias(\"is_mid_month\"),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:07.321948Z","iopub.execute_input":"2026-06-28T09:01:07.322749Z","iopub.status.idle":"2026-06-28T09:01:07.328839Z","shell.execute_reply.started":"2026-06-28T09:01:07.322715Z","shell.execute_reply":"2026-06-28T09:01:07.328066Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Tổng hợp bảng các thông tin:\n- train_static_feats\n- train_credit_bureau_trend\n- train_credit_exposure\n- train_fraud_signals\n- train_tax_stability\n- train_credit_bureau_b_2_enhanced\n- train_collateral ","metadata":{}},{"cell_type":"code","source":"# Dùng collect_schema().names() thay vì .columns trên LazyFrame\nselected_static_cols = [\n    col for col in train_static.collect_schema().names()\n    if col[-1] in (\"A\", \"M\")\n]\nprint(selected_static_cols)\n\nselected_static_cb_cols = [\n    col for col in train_static_cb.collect_schema().names()\n    if col[-1] in (\"A\", \"M\")\n]\nprint(selected_static_cb_cols)\n\n# Join all tables together\ndata = train_basetable.join(\n    train_static.select([\"case_id\"] + selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_static_cb.select([\"case_id\"] + selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    train_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    train_static_feats, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_trend, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_exposure, how=\"left\", on=\"case_id\"\n).join(\n    train_fraud_signals, how=\"left\", on=\"case_id\"\n).join(\n    train_tax_stability, how=\"left\", on=\"case_id\"\n).join(\n    train_credit_bureau_b_2_enhanced, how=\"left\", on=\"case_id\"\n).join(\n    train_collateral, how=\"left\", on=\"case_id\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:10.966333Z","iopub.execute_input":"2026-06-28T09:01:10.966666Z","iopub.status.idle":"2026-06-28T09:01:10.976111Z","shell.execute_reply.started":"2026-06-28T09:01:10.966638Z","shell.execute_reply":"2026-06-28T09:01:10.975350Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndata_submission = test_basetable.join(\n    test_static.select([\"case_id\"]+selected_static_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_static_cb.select([\"case_id\"]+selected_static_cb_cols), how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_1, how=\"left\", on=\"case_id\"\n).join(\n    test_person_1_feats_2, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_feats, how=\"left\", on=\"case_id\"\n).join(\n    test_static_feats, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_trend, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_exposure, how=\"left\", on=\"case_id\"\n).join(\n    test_fraud_signals, how=\"left\", on=\"case_id\"\n).join(\n    test_tax_stability, how=\"left\", on=\"case_id\"\n).join(\n    test_credit_bureau_b_2_enhanced, how=\"left\", on=\"case_id\"\n).join(\n    test_collateral, how=\"left\", on=\"case_id\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:12.172980Z","iopub.execute_input":"2026-06-28T09:01:12.173639Z","iopub.status.idle":"2026-06-28T09:01:12.180171Z","shell.execute_reply.started":"2026-06-28T09:01:12.173604Z","shell.execute_reply":"2026-06-28T09:01:12.179282Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Collect nếu data còn là LazyFrame\nif isinstance(data, pl.LazyFrame):\n    data = data.collect()\n\n# Split case_id\ncase_ids = data[\"case_id\"].unique().shuffle(seed=1).to_list()\n\ncase_ids_train, case_ids_test = train_test_split(\n    case_ids,\n    train_size=0.6,\n    random_state=1\n)\n\ncase_ids_valid, case_ids_test = train_test_split(\n    case_ids_test,\n    train_size=0.5,\n    random_state=1\n)\n\n# Lấy tất cả feature, chỉ bỏ các cột metadata / label\nexcluded_cols = {\n    \"case_id\",\n    \"target\",\n    \"WEEK_NUM\",\n    \"date_decision\",\n}\n\ncols_pred = [\n    col for col in data.columns\n    if col not in excluded_cols\n]\n\nprint(\"Number of features:\", len(cols_pred))\nprint(cols_pred)\n\ndef from_polars_to_pandas(case_ids):\n    df = data.filter(\n        pl.col(\"case_id\").is_in(case_ids)\n    )\n\n    base = df[[\"case_id\", \"WEEK_NUM\", \"target\"]].to_pandas()\n    X = df[cols_pred].to_pandas()\n    y = df[\"target\"].to_pandas()\n\n    return base, X, y\n\nbase_train, X_train, y_train = from_polars_to_pandas(case_ids_train)\nbase_valid, X_valid, y_valid = from_polars_to_pandas(case_ids_valid)\nbase_test, X_test, y_test = from_polars_to_pandas(case_ids_test)\n\n# Convert object/string sang category\nX_train = convert_strings(X_train)\nX_valid = convert_strings(X_valid)\nX_test = convert_strings(X_test)\n\nprint(f\"Train: {X_train.shape}\")\nprint(f\"Valid: {X_valid.shape}\")\nprint(f\"Test : {X_test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:15.913313Z","iopub.execute_input":"2026-06-28T09:01:15.914080Z","iopub.status.idle":"2026-06-28T09:01:40.726971Z","shell.execute_reply.started":"2026-06-28T09:01:15.914047Z","shell.execute_reply":"2026-06-28T09:01:40.725823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Xgboost \n\nTrong phần này chỉ sử dụng xgboost để thử nghiệm train","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import roc_auc_score\n\n\ndef prepare_xgb_data(X_train, X_valid, X_test, X_submission=None):\n    X_train = X_train.copy()\n    X_valid = X_valid.copy()\n    X_test = X_test.copy()\n\n    datasets = [X_train, X_valid, X_test]\n    if X_submission is not None:\n        X_submission = X_submission.copy()\n        datasets.append(X_submission)\n\n    cat_cols = X_train.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n\n    for col in cat_cols:\n        all_values = pd.concat(\n            [df[col].astype(\"object\") for df in datasets],\n            axis=0\n        ).fillna(\"Unknown\")\n\n        categories = all_values.astype(\"category\").cat.categories\n\n        dtype = pd.CategoricalDtype(categories=categories)\n\n        for df in datasets:\n            df[col] = df[col].astype(\"object\").fillna(\"Unknown\").astype(dtype).cat.codes\n\n    if X_submission is not None:\n        return X_train, X_valid, X_test, X_submission\n\n    return X_train, X_valid, X_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T08:31:01.607618Z","iopub.execute_input":"2026-06-28T08:31:01.607977Z","iopub.status.idle":"2026-06-28T08:31:02.068766Z","shell.execute_reply.started":"2026-06-28T08:31:01.607949Z","shell.execute_reply":"2026-06-28T08:31:02.067869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_xgb, X_valid_xgb, X_test_xgb = prepare_xgb_data(\n    X_train, X_valid, X_test\n)\n\nmodel = xgb.XGBClassifier(\n    n_estimators=1000,\n    max_depth=3,\n    learning_rate=0.05,\n    subsample=0.8,\n    colsample_bytree=0.9,\n    objective=\"binary:logistic\",\n    eval_metric=\"auc\",\n    device=\"cuda\",\n    tree_method=\"hist\",\n    random_state=1,\n    n_jobs=-1,\n    early_stopping_rounds=50,\n)\n\nmodel.fit(\n    X_train_xgb,\n    y_train,\n    eval_set=[(X_valid_xgb, y_valid)],\n    verbose=50,\n)\n\nvalid_pred = model.predict_proba(X_valid_xgb)[:, 1]\nprint(\"Valid AUC:\", roc_auc_score(y_valid, valid_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T08:31:02.069731Z","iopub.execute_input":"2026-06-28T08:31:02.070032Z","iopub.status.idle":"2026-06-28T08:32:36.230659Z","shell.execute_reply.started":"2026-06-28T08:31:02.070007Z","shell.execute_reply":"2026-06-28T08:32:36.229827Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Evaluation with AUC and then comparison with the stability metric is shown below.","metadata":{}},{"cell_type":"code","source":"datasets = {\n    \"Train\": (base_train, X_train_xgb),\n    \"Valid\": (base_valid, X_valid_xgb),\n    \"Test\": (base_test, X_test_xgb),\n}\n\nfor name, (base, X) in datasets.items():\n    base[\"score\"] = model.predict_proba(X)[:, 1]\n    auc = roc_auc_score(base[\"target\"], base[\"score\"])\n    print(f\"{name:5s} AUC: {auc:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T08:32:36.231774Z","iopub.execute_input":"2026-06-28T08:32:36.232127Z","iopub.status.idle":"2026-06-28T08:32:54.314487Z","shell.execute_reply.started":"2026-06-28T08:32:36.232094Z","shell.execute_reply":"2026-06-28T08:32:54.313794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std\n\nstability_score_train = gini_stability(base_train)\nstability_score_valid = gini_stability(base_valid)\nstability_score_test = gini_stability(base_test)\n\nprint(f'The stability score on the train set is: {stability_score_train}') \nprint(f'The stability score on the valid set is: {stability_score_valid}') \nprint(f'The stability score on the test set is: {stability_score_test}')  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T08:32:54.315365Z","iopub.execute_input":"2026-06-28T08:32:54.315713Z","iopub.status.idle":"2026-06-28T08:32:55.312449Z","shell.execute_reply.started":"2026-06-28T08:32:54.315687Z","shell.execute_reply":"2026-06-28T08:32:55.311715Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Áp dụng cách cải tiển của nhóm phát triển\n\n1. Train LightGBM","metadata":{}},{"cell_type":"code","source":"# Các cột category của train\ncat_cols_lgb = X_train.select_dtypes(include=[\"category\"]).columns.tolist()\n\nprint(f\"Remove {len(cat_cols_lgb)} categorical columns:\")\nprint(cat_cols_lgb)\n\n# Drop đồng bộ\nX_train_lgb = X_train.drop(columns=cat_cols_lgb)\nX_valid_lgb = X_valid.drop(columns=cat_cols_lgb)\nX_test_lgb = X_test.drop(columns=cat_cols_lgb)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:40.728638Z","iopub.execute_input":"2026-06-28T09:01:40.730311Z","iopub.status.idle":"2026-06-28T09:01:42.827715Z","shell.execute_reply.started":"2026-06-28T09:01:40.730254Z","shell.execute_reply":"2026-06-28T09:01:42.827119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop categorical columns cho LightGBM\ncat_cols_lgb = X_train.select_dtypes(include=[\"category\"]).columns.tolist()\n\nprint(f\"Remove {len(cat_cols_lgb)} categorical columns:\")\nprint(cat_cols_lgb)\n\nX_train_lgb = X_train.drop(columns=cat_cols_lgb)\nX_valid_lgb = X_valid.drop(columns=cat_cols_lgb)\nX_test_lgb = X_test.drop(columns=cat_cols_lgb)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:42.828535Z","iopub.execute_input":"2026-06-28T09:01:42.828775Z","iopub.status.idle":"2026-06-28T09:01:44.176974Z","shell.execute_reply.started":"2026-06-28T09:01:42.828753Z","shell.execute_reply":"2026-06-28T09:01:44.176319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_train = lgb.Dataset(\n    X_train_lgb,\n    label=y_train,\n    free_raw_data=False\n)\n\nlgb_valid = lgb.Dataset(\n    X_valid_lgb,\n    label=y_valid,\n    reference=lgb_train,\n    free_raw_data=False\n)\n\nlgb_params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"learning_rate\": 0.03,\n    \"num_leaves\": 64,\n    \"max_depth\": -1,\n    \"feature_fraction\": 0.85,\n    \"bagging_fraction\": 0.85,\n    \"bagging_freq\": 5,\n    \"min_data_in_leaf\": 50,\n    \"lambda_l1\": 0.1,\n    \"lambda_l2\": 5.0,\n    \"verbose\": -1,\n    \"seed\": 1,\n\n    # nếu muốn thử GPU\n    \"device_type\": \"gpu\",\n}\n\nlgb_model = lgb.train(\n    lgb_params,\n    lgb_train,\n    num_boost_round=5000,\n    valid_sets=[lgb_valid],\n    valid_names=[\"valid\"],\n    callbacks=[\n        lgb.log_evaluation(100),\n        lgb.early_stopping(200)\n    ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:01:44.178554Z","iopub.execute_input":"2026-06-28T09:01:44.178858Z","iopub.status.idle":"2026-06-28T09:05:02.521992Z","shell.execute_reply.started":"2026-06-28T09:01:44.178827Z","shell.execute_reply":"2026-06-28T09:05:02.521406Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2. Train XGBoost","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\nX_train_xgb, X_valid_xgb, X_test_xgb = prepare_xgb_data(\n    X_train,\n    X_valid,\n    X_test\n)\n\nxgb_model = xgb.XGBClassifier(\n    n_estimators=5000,\n    learning_rate=0.03,\n    max_depth=5,\n    min_child_weight=10,\n    subsample=0.85,\n    colsample_bytree=0.85,\n    reg_alpha=0.1,\n    reg_lambda=5.0,\n    objective=\"binary:logistic\",\n    eval_metric=\"auc\",\n    tree_method=\"hist\",\n    device=\"cuda\",\n    random_state=1,\n    early_stopping_rounds=200,\n)\n\nxgb_model.fit(\n    X_train_xgb,\n    y_train,\n    eval_set=[(X_valid_xgb, y_valid)],\n    verbose=100\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:05:02.522724Z","iopub.execute_input":"2026-06-28T09:05:02.522927Z","iopub.status.idle":"2026-06-28T09:08:18.781281Z","shell.execute_reply.started":"2026-06-28T09:05:02.522909Z","shell.execute_reply":"2026-06-28T09:08:18.780570Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3. Train CatBoost","metadata":{}},{"cell_type":"code","source":"from catboost import CatBoostClassifier\n\ncat_cols_cat = X_train.select_dtypes(include=[\"category\", \"object\"]).columns.tolist()\ncat_features = [X_train.columns.get_loc(c) for c in cat_cols_cat]\n\nX_train_cat = X_train.copy()\nX_valid_cat = X_valid.copy()\nX_test_cat = X_test.copy()\n\nfor df in [X_train_cat, X_valid_cat, X_test_cat]:\n    for col in cat_cols_cat:\n        df[col] = df[col].astype(\"object\").fillna(\"Unknown\").astype(str)\n\ncat_model = CatBoostClassifier(\n    iterations=6000,\n    learning_rate=0.03,\n    depth=6,\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=1,\n    task_type=\"GPU\",\n    devices=\"0\",\n    verbose=100,\n    early_stopping_rounds=200,\n)\n\ncat_model.fit(\n    X_train_cat,\n    y_train,\n    eval_set=(X_valid_cat, y_valid),\n    cat_features=cat_features,\n    use_best_model=True\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:08:18.782381Z","iopub.execute_input":"2026-06-28T09:08:18.782694Z","iopub.status.idle":"2026-06-28T09:23:47.356389Z","shell.execute_reply.started":"2026-06-28T09:08:18.782663Z","shell.execute_reply":"2026-06-28T09:23:47.355519Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"4. Dự đoán validation/test cho từng model","metadata":{}},{"cell_type":"code","source":"pred_lgb_train = lgb_model.predict(\n    X_train_lgb,\n    num_iteration=lgb_model.best_iteration\n)\n\npred_lgb_valid = lgb_model.predict(\n    X_valid_lgb,\n    num_iteration=lgb_model.best_iteration\n)\n\npred_lgb_test = lgb_model.predict(\n    X_test_lgb,\n    num_iteration=lgb_model.best_iteration\n)\n\n# =====================\n# XGBoost predictions\n# =====================\npred_xgb_train = xgb_model.predict_proba(X_train_xgb)[:, 1]\npred_xgb_valid = xgb_model.predict_proba(X_valid_xgb)[:, 1]\npred_xgb_test = xgb_model.predict_proba(X_test_xgb)[:, 1]\n\n# =====================\n# CatBoost predictions\n# =====================\npred_cat_train = cat_model.predict_proba(X_train_cat)[:, 1]\npred_cat_valid = cat_model.predict_proba(X_valid_cat)[:, 1]\npred_cat_test = cat_model.predict_proba(X_test_cat)[:, 1]\n\n# =====================\n# AUC từng model\n# =====================\nprint(\"=\" * 60)\nprint(\"AUC SCORE\")\nprint(\"=\" * 60)\n\nprint(f\"LGB Train AUC : {roc_auc_score(y_train, pred_lgb_train):.6f}\")\nprint(f\"LGB Valid AUC : {roc_auc_score(y_valid, pred_lgb_valid):.6f}\")\nprint(f\"LGB Test  AUC : {roc_auc_score(y_test, pred_lgb_test):.6f}\")\nprint(\"-\" * 60)\n\nprint(f\"XGB Train AUC : {roc_auc_score(y_train, pred_xgb_train):.6f}\")\nprint(f\"XGB Valid AUC : {roc_auc_score(y_valid, pred_xgb_valid):.6f}\")\nprint(f\"XGB Test  AUC : {roc_auc_score(y_test, pred_xgb_test):.6f}\")\nprint(\"-\" * 60)\n\nprint(f\"CAT Train AUC : {roc_auc_score(y_train, pred_cat_train):.6f}\")\nprint(f\"CAT Valid AUC : {roc_auc_score(y_valid, pred_cat_valid):.6f}\")\nprint(f\"CAT Test  AUC : {roc_auc_score(y_test, pred_cat_test):.6f}\")\nprint(\"=\" * 60)\n\n# =====================\n# Stability từng model\n# =====================\ndef evaluate_stability(base, pred):\n    tmp = base.copy()\n    tmp[\"score\"] = pred\n    return gini_stability(tmp)\n\nprint(\"GINI STABILITY\")\nprint(\"=\" * 60)\n\nprint(f\"LGB Train Stability : {evaluate_stability(base_train, pred_lgb_train):.6f}\")\nprint(f\"LGB Valid Stability : {evaluate_stability(base_valid, pred_lgb_valid):.6f}\")\nprint(f\"LGB Test  Stability : {evaluate_stability(base_test, pred_lgb_test):.6f}\")\nprint(\"-\" * 60)\n\nprint(f\"XGB Train Stability : {evaluate_stability(base_train, pred_xgb_train):.6f}\")\nprint(f\"XGB Valid Stability : {evaluate_stability(base_valid, pred_xgb_valid):.6f}\")\nprint(f\"XGB Test  Stability : {evaluate_stability(base_test, pred_xgb_test):.6f}\")\nprint(\"-\" * 60)\n\nprint(f\"CAT Train Stability : {evaluate_stability(base_train, pred_cat_train):.6f}\")\nprint(f\"CAT Valid Stability : {evaluate_stability(base_valid, pred_cat_valid):.6f}\")\nprint(f\"CAT Test  Stability : {evaluate_stability(base_test, pred_cat_test):.6f}\")\nprint(\"=\" * 60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:23:47.358283Z","iopub.execute_input":"2026-06-28T09:23:47.358646Z","iopub.status.idle":"2026-06-28T09:26:52.499369Z","shell.execute_reply.started":"2026-06-28T09:23:47.358622Z","shell.execute_reply":"2026-06-28T09:26:52.498691Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"5. Weighted Ensemble","metadata":{}},{"cell_type":"code","source":"weights = {\n    \"lgb\": 0.40,\n    \"xgb\": 0.35,\n    \"cat\": 0.25,\n}\n\npred_valid_ens = (\n    weights[\"lgb\"] * pred_lgb_valid +\n    weights[\"xgb\"] * pred_xgb_valid +\n    weights[\"cat\"] * pred_cat_valid\n)\n\nprint(\"Ensemble Valid AUC:\", roc_auc_score(y_valid, pred_valid_ens))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:27:05.095211Z","iopub.execute_input":"2026-06-28T09:27:05.095510Z","iopub.status.idle":"2026-06-28T09:27:05.215457Z","shell.execute_reply.started":"2026-06-28T09:27:05.095485Z","shell.execute_reply":"2026-06-28T09:27:05.214597Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"6. Tìm trọng số tốt nhất đơn giản","metadata":{}},{"cell_type":"code","source":"best_stability = -np.inf\nbest_auc = -np.inf\nbest_weights = None\n\nfor w_lgb in np.arange(0.0, 1.01, 0.05):\n    for w_xgb in np.arange(0.0, 1.01 - w_lgb, 0.05):\n        w_cat = round(1.0 - w_lgb - w_xgb, 2)\n\n        pred = (\n            w_lgb * pred_lgb_valid +\n            w_xgb * pred_xgb_valid +\n            w_cat * pred_cat_valid\n        )\n\n        auc = roc_auc_score(y_valid, pred)\n\n        tmp = base_valid.copy()\n        tmp[\"score\"] = pred\n\n        stability = gini_stability(tmp)\n\n        # Ưu tiên Stability, nếu bằng nhau thì chọn AUC lớn hơn\n        if (\n            stability > best_stability or\n            (np.isclose(stability, best_stability) and auc > best_auc)\n        ):\n            best_stability = stability\n            best_auc = auc\n            best_weights = (w_lgb, w_xgb, w_cat)\n\nprint(\"=\" * 60)\nprint(f\"Best Stability : {best_stability:.6f}\")\nprint(f\"Best AUC       : {best_auc:.6f}\")\nprint(\n    f\"Best Weights   : \"\n    f\"LGB={best_weights[0]:.2f}, \"\n    f\"XGB={best_weights[1]:.2f}, \"\n    f\"CAT={best_weights[2]:.2f}\"\n)\nprint(\"=\" * 60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:27:07.623497Z","iopub.execute_input":"2026-06-28T09:27:07.623780Z","iopub.status.idle":"2026-06-28T09:28:31.533259Z","shell.execute_reply.started":"2026-06-28T09:27:07.623758Z","shell.execute_reply":"2026-06-28T09:28:31.532571Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"7. Đánh giá train/valid/test bằng ensemble","metadata":{}},{"cell_type":"code","source":"w_lgb, w_xgb, w_cat = best_weights\n\npred_lgb_train = lgb_model.predict(\n    X_train_lgb,\n    num_iteration=lgb_model.best_iteration\n)\npred_lgb_valid = lgb_model.predict(\n    X_valid_lgb,\n    num_iteration=lgb_model.best_iteration\n)\npred_lgb_test = lgb_model.predict(\n    X_test_lgb,\n    num_iteration=lgb_model.best_iteration\n)\n\npred_xgb_train = xgb_model.predict_proba(X_train_xgb)[:, 1]\npred_xgb_valid = xgb_model.predict_proba(X_valid_xgb)[:, 1]\npred_xgb_test = xgb_model.predict_proba(X_test_xgb)[:, 1]\n\npred_cat_train = cat_model.predict_proba(X_train_cat)[:, 1]\npred_cat_valid = cat_model.predict_proba(X_valid_cat)[:, 1]\npred_cat_test = cat_model.predict_proba(X_test_cat)[:, 1]\n\nbase_train = base_train.copy()\nbase_valid = base_valid.copy()\nbase_test = base_test.copy()\n\nbase_train[\"score\"] = (\n    w_lgb * pred_lgb_train +\n    w_xgb * pred_xgb_train +\n    w_cat * pred_cat_train\n)\n\nbase_valid[\"score\"] = (\n    w_lgb * pred_lgb_valid +\n    w_xgb * pred_xgb_valid +\n    w_cat * pred_cat_valid\n)\n\nbase_test[\"score\"] = (\n    w_lgb * pred_lgb_test +\n    w_xgb * pred_xgb_test +\n    w_cat * pred_cat_test\n)\n\nprint(f\"Train AUC: {roc_auc_score(base_train['target'], base_train['score']):.6f}\")\nprint(f\"Valid AUC: {roc_auc_score(base_valid['target'], base_valid['score']):.6f}\")\nprint(f\"Test  AUC: {roc_auc_score(base_test['target'], base_test['score']):.6f}\")\n\nprint(f\"Train Stability: {gini_stability(base_train):.6f}\")\nprint(f\"Valid Stability: {gini_stability(base_valid):.6f}\")\nprint(f\"Test  Stability: {gini_stability(base_test):.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:28:31.539707Z","iopub.execute_input":"2026-06-28T09:28:31.540055Z","iopub.status.idle":"2026-06-28T09:31:32.636254Z","shell.execute_reply.started":"2026-06-28T09:28:31.540031Z","shell.execute_reply":"2026-06-28T09:31:32.635382Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission\n\nScoring the submission dataset is below, we need to take care of new categories. Then we save the score as a last step. ","metadata":{}},{"cell_type":"code","source":"if isinstance(data_submission, pl.LazyFrame):\n    data_submission = data_submission.collect()\n\n# Bảo đảm submission có đủ cols_pred\nmissing_cols = [c for c in cols_pred if c not in data_submission.columns]\nfor c in missing_cols:\n    data_submission = data_submission.with_columns(pl.lit(None).alias(c))\n\n# Base submission dataframe\nX_submission = data_submission[cols_pred].to_pandas()\nX_submission = convert_strings(X_submission)\nX_submission_lgb = X_submission.drop(columns=cat_cols_lgb)\n\n# =====================\n# LightGBM\n# =====================\nlgb_features = X_train_lgb.columns.tolist()\n\nX_submission_lgb = X_submission.reindex(columns=lgb_features)\n\nfor col in X_submission_lgb.columns:\n    X_submission_lgb[col] = pd.to_numeric(\n        X_submission_lgb[col],\n        errors=\"coerce\"\n    )\n\nX_submission_lgb = X_submission_lgb.fillna(-1).astype(\"float32\")\n\nprint(\n    \"LGB bad dtypes:\",\n    X_submission_lgb.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n)\n\n# =====================\n# XGBoost\n# =====================\n_, _, _, X_submission_xgb = prepare_xgb_data(\n    X_train,\n    X_valid,\n    X_test,\n    X_submission,\n)\n\n# Bảo đảm XGBoost không còn object/category\nbad_xgb_cols = X_submission_xgb.select_dtypes(\n    include=[\"object\", \"category\"]\n).columns.tolist()\n\nfor col in bad_xgb_cols:\n    X_submission_xgb[col] = (\n        X_submission_xgb[col]\n        .astype(\"category\")\n        .cat.codes\n        .astype(\"int32\")\n    )\n\nX_submission_xgb = X_submission_xgb.reindex(columns=X_train_xgb.columns)\n\nprint(\n    \"XGB bad dtypes:\",\n    X_submission_xgb.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n)\n\n# =====================\n# CatBoost\n# =====================\ncat_features_all = X_train_cat.columns.tolist()\n\nX_submission_cat = X_submission.reindex(columns=cat_features_all)\n\nfor col in cat_cols_cat:\n    if col in X_submission_cat.columns:\n        X_submission_cat[col] = (\n            X_submission_cat[col]\n            .astype(\"object\")\n            .fillna(\"Unknown\")\n            .astype(str)\n        )\n\n# Các cột không phải categorical thì ép numeric\nfor col in X_submission_cat.columns:\n    if col not in cat_cols_cat:\n        X_submission_cat[col] = pd.to_numeric(\n            X_submission_cat[col],\n            errors=\"coerce\"\n        ).fillna(-1)\n\n# =====================\n# Predict\n# =====================\npred_lgb_sub = lgb_model.predict(\n    X_submission_lgb,\n    num_iteration=lgb_model.best_iteration\n)\n\npred_xgb_sub = xgb_model.predict_proba(X_submission_xgb)[:, 1]\n\npred_cat_sub = cat_model.predict_proba(X_submission_cat)[:, 1]\n\ny_submission_pred = (\n    w_lgb * pred_lgb_sub +\n    w_xgb * pred_xgb_sub +\n    w_cat * pred_cat_sub\n)\n\nprint(\"Submission prediction shape:\", y_submission_pred.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:31:32.638006Z","iopub.execute_input":"2026-06-28T09:31:32.638359Z","iopub.status.idle":"2026-06-28T09:31:57.128445Z","shell.execute_reply.started":"2026-06-28T09:31:32.638327Z","shell.execute_reply":"2026-06-28T09:31:57.127521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = data_submission[[\"case_id\"]].to_pandas()\nsubmission[\"score\"] = y_submission_pred\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-28T09:31:57.130060Z","iopub.execute_input":"2026-06-28T09:31:57.130425Z","iopub.status.idle":"2026-06-28T09:31:57.190938Z","shell.execute_reply.started":"2026-06-28T09:31:57.130401Z","shell.execute_reply":"2026-06-28T09:31:57.190228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}