{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":12170976,"sourceType":"datasetVersion","datasetId":7665406},{"sourceId":12173496,"sourceType":"datasetVersion","datasetId":7667052},{"sourceId":12174382,"sourceType":"datasetVersion","datasetId":7667537},{"sourceId":12181987,"sourceType":"datasetVersion","datasetId":7672599},{"sourceId":12186716,"sourceType":"datasetVersion","datasetId":7675947},{"sourceId":12186731,"sourceType":"datasetVersion","datasetId":7675959},{"sourceId":12186933,"sourceType":"datasetVersion","datasetId":7676108},{"sourceId":12186982,"sourceType":"datasetVersion","datasetId":7676149},{"sourceId":12188560,"sourceType":"datasetVersion","datasetId":7677224}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars gosdt","metadata":{"_uuid":"431b4b05-9cb4-4977-977d-34a5cc00859b","_cell_guid":"21a4d638-7619-4ce8-ad65-3fa0fbec564c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:38:36.047546Z","iopub.execute_input":"2025-06-23T20:38:36.047837Z","iopub.status.idle":"2025-06-23T20:38:44.518233Z","shell.execute_reply.started":"2025-06-23T20:38:36.047814Z","shell.execute_reply":"2025-06-23T20:38:44.513364Z"},"_kg_hide-input":true,"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"Collecting polars\n Downloading polars-1.31.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (35.1 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m35.1/35.1 MB\u001b[0m \u001b[31m28.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[?25hCollecting gosdt\n Downloading gosdt-1.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m60.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: pandas>=2.0.1 in /usr/local/lib/python3.10/site-packages (from gosdt) (2.2.3)\nRequirement already satisfied: numpy>=1.24.3 in /usr/local/lib/python3.10/site-packages (from gosdt) (2.0.2)\nRequirement already satisfied: scikit-learn>=1.2.2 in /usr/local/lib/python3.10/site-packages (from gosdt) (1.6.1)\nRequirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas>=2.0.1->gosdt) (2.9.0.post0)\nRequirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas>=2.0.1->gosdt) (2025.2)\nRequirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/site-packages (from pandas>=2.0.1->gosdt) (2025.2)\nRequirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=1.2.2->gosdt) (1.15.2)\nRequirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=1.2.2->gosdt) (3.6.0)\nRequirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/site-packages (from scikit-learn>=1.2.2->gosdt) (1.5.0)\nRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=2.0.1->gosdt) (1.17.0)\nInstalling collected packages: polars, gosdt\nSuccessfully installed gosdt-1.0.4 polars-1.31.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.1.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}],"execution_count":1},{"cell_type":"code","source":"import gosdt\nprint(\"GOSDT is at:\", gosdt.__file__)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:38:44.521662Z","iopub.execute_input":"2025-06-23T20:38:44.521943Z","iopub.status.idle":"2025-06-23T20:38:46.45669Z","shell.execute_reply.started":"2025-06-23T20:38:44.521914Z","shell.execute_reply":"2025-06-23T20:38:46.453499Z"}},"outputs":[{"name":"stdout","text":"GOSDT is at: /usr/local/lib/python3.10/site-packages/gosdt/__init__.py\n","output_type":"stream"}],"execution_count":2},{"cell_type":"code","source":"import gosdt\nimport inspect, os\n\n# Đường dẫn thư mục chứa gosdt\npkg_dir = os.path.dirname(inspect.getsourcefile(gosdt))\nprint(\"Package dir:\", pkg_dir)\n\n# Liệt kê các thuộc tính public ở cấp top-level\nattrs = [a for a in dir(gosdt) if not a.startswith(\"_\")]\nfor name in attrs:\n obj = getattr(gosdt, name)\n src = None\n try:\n src = inspect.getsourcefile(obj)\n except (TypeError, OSError):\n pass\n print(f\"{name:20s} type={type(obj).__name__:12s} source_file={src}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:38:46.45948Z","iopub.execute_input":"2025-06-23T20:38:46.459833Z","iopub.status.idle":"2025-06-23T20:38:46.471983Z","shell.execute_reply.started":"2025-06-23T20:38:46.459809Z","shell.execute_reply":"2025-06-23T20:38:46.466814Z"}},"outputs":[{"name":"stdout","text":"Package dir: /usr/local/lib/python3.10/site-packages/gosdt\nGOSDTClassifier type=type source_file=/usr/local/lib/python3.10/site-packages/gosdt/_classifier.py\nNumericBinarizer type=type source_file=/usr/local/lib/python3.10/site-packages/gosdt/_binarizer.py\nStatus type=pybind11_type source_file=None\nThresholdGuessBinarizer type=type source_file=/usr/local/lib/python3.10/site-packages/gosdt/_threshold_guessing.py\nannotations type=_Feature source_file=None\n","output_type":"stream"}],"execution_count":3},{"cell_type":"code","source":"# with open('/usr/local/lib/python3.11/dist-packages/gosdt/_threshold_guessing.py', 'r', encoding='utf-8') as f:\n# print(f.read())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:38:46.473779Z","iopub.execute_input":"2025-06-23T20:38:46.474019Z","iopub.status.idle":"2025-06-23T20:38:46.498713Z","shell.execute_reply.started":"2025-06-23T20:38:46.473998Z","shell.execute_reply":"2025-06-23T20:38:46.494425Z"}},"outputs":[],"execution_count":4},{"cell_type":"code","source":"import pandas as pd\ntrain_base = pd.read_parquet('/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/train/train_base.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:38:46.499498Z","iopub.execute_input":"2025-06-23T20:38:46.499726Z","iopub.status.idle":"2025-06-23T20:38:46.803583Z","shell.execute_reply.started":"2025-06-23T20:38:46.499705Z","shell.execute_reply":"2025-06-23T20:38:46.799954Z"}},"outputs":[],"execution_count":5},{"cell_type":"raw","source":"train_base.to_csv('train_base.csv')","metadata":{}},{"cell_type":"code","source":"train_base.to_csv('train_base.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:38:46.806444Z","iopub.execute_input":"2025-06-23T20:38:46.806698Z","iopub.status.idle":"2025-06-23T20:38:49.904749Z","shell.execute_reply.started":"2025-06-23T20:38:46.806676Z","shell.execute_reply":"2025-06-23T20:38:49.899911Z"}},"outputs":[],"execution_count":6},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Import Libraries","metadata":{"_uuid":"dc30ca95-d3f4-477d-a528-5fc6d78b81ee","_cell_guid":"38754549-8dda-445e-8e6f-6e640036d574","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport polars as pl","metadata":{"_uuid":"64525f2b-78bb-4c2e-8872-1aa8d4fc49d4","_cell_guid":"50f86e11-155b-405f-8904-9098cb27a0dd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:38:49.906905Z","iopub.execute_input":"2025-06-23T20:38:49.90713Z","iopub.status.idle":"2025-06-23T20:38:52.080811Z","shell.execute_reply.started":"2025-06-23T20:38:49.907109Z","shell.execute_reply":"2025-06-23T20:38:52.076669Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":7},{"cell_type":"markdown","source":"# 2. Configuration","metadata":{"_uuid":"57dc07a1-9947-40f8-940c-eb2123a57353","_cell_guid":"4a2bd8c0-1ae9-4835-b257-4e5630dcb029","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# import warnings\n\n# # Pandas configuration\n# pd.set_option('display.max_columns', None)\n# pd.set_option('display.max_rows',100)\n# pd.set_option('display.float_format',lambda x: '%.3f'%x)\n\n# # Seaborn configuration\n# sns.set_palette('viridis')\n# sns.set_context('notebook')\n\n# # Warning turning off\n# warnings.filterwarnings('ignore')\n\n# # Matplotlib configuration\n# %matplotlib inline\n# plt.style.use('seaborn-whitegrid')\n# plt.rcParams['figure.figsize'] = (12,6)\n# plt.rcParams['figure.dpi'] = 100\n# plt.rcParams['font.size'] = 14\n# plt.rcParams['axes.labelsize'] = 14\n# plt.rcParams['axes.titlesize'] = 18\n\n# # Reload Automation lib\n# %load_ext autoreload\n# %autoreload 2\n\n# import logging\n# logger = logging.getLogger(\"py4j\")\n# logger.setLevel(logging.ERROR)","metadata":{"_uuid":"5698fa5f-e220-4f48-b6b9-435c4696fb48","_cell_guid":"0467f3b3-3158-480e-b40c-0418e22b8242","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:38:52.083361Z","iopub.execute_input":"2025-06-23T20:38:52.083744Z","iopub.status.idle":"2025-06-23T20:38:52.092738Z","shell.execute_reply.started":"2025-06-23T20:38:52.08372Z","shell.execute_reply":"2025-06-23T20:38:52.088519Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":8},{"cell_type":"markdown","source":"# 3. Read file","metadata":{"_uuid":"61294b54-287f-4d06-85f1-e03879b9d021","_cell_guid":"8267cd59-fc52-46f6-8ff8-f1a2f119da35","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import polars as pl","metadata":{"_uuid":"7736f790-49b0-4b9b-9f5c-602fee2394c9","_cell_guid":"7306e661-ea76-4eeb-9f96-2186b5c75e7f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:38:52.096374Z","iopub.execute_input":"2025-06-23T20:38:52.096618Z","iopub.status.idle":"2025-06-23T20:38:52.110147Z","shell.execute_reply.started":"2025-06-23T20:38:52.096595Z","shell.execute_reply":"2025-06-23T20:38:52.105748Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":9},{"cell_type":"code","source":"import polars as pl\n\n# Đọc train_base và xóa date_decision\ntrain_base = pl.read_csv('/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/train_base.csv')\ntrain_base = train_base.drop(\"date_decision\")\n\n# Định nghĩa hàm thêm cột has_feature\ndef add_has_flag(df: pl.DataFrame, feature_name: str) -> pl.DataFrame:\n return df.with_columns([\n pl.lit(1).alias(f\"has_{feature_name}\")\n ])\n\n# Đọc và gán cột has_* cho từng bảng\napplprev_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/previous application features.csv'), \"applprev\")\ncredit_bureau_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/credit_features_complete.csv'), \"credit_bureau\")\ntax_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/tax_features_extended.csv'), \"tax\")\nperson_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/person_feature.csv'), \"person\")\ndeposit_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/deposit_feature.csv'), \"deposit\")\nstatic_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/static_0_feature.csv'), \"static\")\nstatic_cb_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/static_cb_features.csv'), \"static_cb\")\nother_features = add_has_flag(pl.read_csv('/kaggle/input/home-features/other_feature.csv'), \"other\")\n\n# Thực hiện join từng bảng (với cột has_*)\ntrain_merged = (\n train_base\n .join(applprev_features, on=\"case_id\", how=\"left\")\n .join(credit_bureau_features, on=\"case_id\", how=\"left\")\n .join(tax_features, on=\"case_id\", how=\"left\")\n .join(person_features, on=\"case_id\", how=\"left\")\n .join(deposit_features, on=\"case_id\", how=\"left\")\n .join(static_features, on=\"case_id\", how=\"left\")\n .join(static_cb_features, on=\"case_id\", how=\"left\")\n .join(other_features, on=\"case_id\", how=\"left\")\n)","metadata":{"_uuid":"40ff89a0-1dfc-46fe-a42c-0c812b06e258","_cell_guid":"327f4c4f-a581-4a2b-81d5-bc9cccc564d4","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:38:52.111826Z","iopub.execute_input":"2025-06-23T20:38:52.112064Z","iopub.status.idle":"2025-06-23T20:39:44.904141Z","shell.execute_reply.started":"2025-06-23T20:38:52.112041Z","shell.execute_reply":"2025-06-23T20:39:44.899941Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":10},{"cell_type":"code","source":"train_merged.schema","metadata":{"_uuid":"427a4bab-4134-4a84-9248-1393b801a5c0","_cell_guid":"4859b89f-a9a1-4246-9e85-a295d599b31a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:44.906081Z","iopub.execute_input":"2025-06-23T20:39:44.906293Z","iopub.status.idle":"2025-06-23T20:39:44.945798Z","shell.execute_reply.started":"2025-06-23T20:39:44.906271Z","shell.execute_reply":"2025-06-23T20:39:44.940501Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"Schema([('case_id', Int64),\n ('MONTH', Int64),\n ('WEEK_NUM', Int64),\n ('target', Int64),\n ('num_applications', Int64),\n ('num_approved', Int64),\n ('rate_approval_application', Float64),\n ('actualdpd_943P_approved_max', Float64),\n ('actualdpd_943P_approved_avg', Float64),\n ('actualdpd_943P_approved_min', Float64),\n ('actualdpd_943P_approved_std', String),\n ('actualdpd_943P_approved_last', Float64),\n ('annuity_853A_approved_max', Float64),\n ('annuity_853A_approved_avg', Float64),\n ('annuity_853A_approved_min', Float64),\n ('annuity_853A_approved_std', String),\n ('annuity_853A_approved_last', Float64),\n ('annuity_853A_rejected_max', Float64),\n ('annuity_853A_rejected_avg', Float64),\n ('annuity_853A_rejected_min', Float64),\n ('annuity_853A_rejected_std', Float64),\n ('annuity_853A_rejected_last', Float64),\n ('annuity_853A_last', Float64),\n ('approvaldate_319D_duration', Int64),\n ('approvaldate_319D_interval', Float64),\n ('byoccupationinc_3656910L_max', Float64),\n ('byoccupationinc_3656910L_avg', Float64),\n ('byoccupationinc_3656910L_min', Float64),\n ('byoccupationinc_3656910L_std', Float64),\n ('byoccupationinc_3656910L_last', Float64),\n ('cancelreason_3545846M_approved_mode', String),\n ('cancelreason_3545846M_rejected_mode', String),\n ('cancelreason_3545846M_overall_mode', String),\n ('cancelreason_3545846M_last', String),\n ('childnum_21L_approved_avg', String),\n ('childnum_21L_rejected_avg', Float64),\n ('childnum_21L_last', Float64),\n ('creationdate_885D_duration_all', Int64),\n ('creationdate_885D_interval_approved', Float64),\n ('creationdate_885D_interval_rejected', Float64),\n ('credacc_credlmt_575A_approved_max', Float64),\n ('credacc_credlmt_575A_approved_avg', Float64),\n ('credacc_credlmt_575A_approved_min', Float64),\n ('credacc_credlmt_575A_approved_std', String),\n ('credacc_credlmt_575A_approved_last', Float64),\n ('credacc_credlmt_575A_rejected_max', Float64),\n ('credacc_credlmt_575A_rejected_avg', Float64),\n ('credacc_credlmt_575A_rejected_min', Float64),\n ('credacc_credlmt_575A_rejected_std', Float64),\n ('credacc_credlmt_575A_rejected_last', Float64),\n ('credacc_credlmt_575A_last', Float64),\n ('credamount_590A_approved_max', Float64),\n ('credamount_590A_approved_avg', Float64),\n ('credamount_590A_approved_min', Float64),\n ('credamount_590A_approved_std', String),\n ('credamount_590A_approved_last', Float64),\n ('credamount_590A_rejected_max', Float64),\n ('credamount_590A_rejected_avg', Float64),\n ('credamount_590A_rejected_min', Float64),\n ('credamount_590A_rejected_std', Float64),\n ('credamount_590A_rejected_last', Float64),\n ('credamount_590A_last', Float64),\n ('credtype_587L_mode', String),\n ('credtype_587L_last', String),\n ('currdebt_94A_approved_max', Float64),\n ('currdebt_94A_approved_avg', Float64),\n ('currdebt_94A_approved_min', Float64),\n ('currdebt_94A_approved_std', String),\n ('currdebt_94A_approved_last', Float64),\n ('currdebt_94A_rejected_max', Float64),\n ('currdebt_94A_rejected_avg', Float64),\n ('currdebt_94A_rejected_min', Float64),\n ('currdebt_94A_rejected_std', Float64),\n ('currdebt_94A_rejected_last', Float64),\n ('currdebt_94A_last', Float64),\n ('district_544M_mode', String),\n ('district_544M_last', String),\n ('downpmt_134A_approved_max', Float64),\n ('downpmt_134A_approved_avg', Float64),\n ('downpmt_134A_approved_min', Float64),\n ('downpmt_134A_approved_std', String),\n ('downpmt_134A_approved_last', Float64),\n ('downpmt_134A_rejected_max', Float64),\n ('downpmt_134A_rejected_avg', Float64),\n ('downpmt_134A_rejected_min', Float64),\n ('downpmt_134A_rejected_std', Float64),\n ('downpmt_134A_rejected_last', Float64),\n ('downpmt_134A_last', Float64),\n ('rate_downpmt_and_credamount_approved_max', Float64),\n ('rate_downpmt_and_credamount_approved_avg', Float64),\n ('rate_downpmt_and_credamount_approved_min', Float64),\n ('rate_downpmt_and_credamount_approved_std', String),\n ('rate_downpmt_and_credamount_approved_last', Float64),\n ('rate_downpmt_and_credamount_rejected_max', Float64),\n ('rate_downpmt_and_credamount_rejected_avg', Float64),\n ('rate_downpmt_and_credamount_rejected_min', Float64),\n ('rate_downpmt_and_credamount_rejected_std', Float64),\n ('rate_downpmt_and_credamount_rejected_last', Float64),\n ('rate_downpmt_and_credamount_last', Float64),\n ('education_1138M_mode', String),\n ('education_1138M_last', String),\n ('employedfrom_creationdate_last', Int64),\n ('employedfrom_creationdate_approved_avg', Float64),\n ('employedfrom_creationdate_approved_max', Int64),\n ('employedfrom_creationdate_approved_min', Int64),\n ('employedfrom_creationdate_approved_std', String),\n ('employedfrom_creationdate_approved_last', Int64),\n ('employedfrom_creationdate_rejected_avg', Float64),\n ('employedfrom_creationdate_rejected_max', Int64),\n ('employedfrom_creationdate_rejected_min', Int64),\n ('employedfrom_creationdate_rejected_std', Float64),\n ('employedfrom_creationdate_rejected_last', Int64),\n ('familystate_726L_mode', String),\n ('familystate_726L_last', String),\n ('diff_firstnonzeroinstldate_creationdate_approved_last', Int64),\n ('diff_firstnonzeroinstldate_creationdate_approved_avg', Float64),\n ('diff_firstnonzeroinstldate_creationdate_approved_max', Int64),\n ('diff_firstnonzeroinstldate_creationdate_approved_min', Int64),\n ('diff_firstnonzeroinstldate_creationdate_approved_std', String),\n ('diff_firstnonzeroinstldate_creationdate_rejected_avg', Float64),\n ('diff_firstnonzeroinstldate_creationdate_rejected_max', Int64),\n ('diff_firstnonzeroinstldate_creationdate_rejected_min', Int64),\n ('diff_firstnonzeroinstldate_creationdate_rejected_std', Float64),\n ('inittransactioncode_279L_mode', String),\n ('inittransactioncode_279L_last', String),\n ('isbidproduct_390L_approved_count', Int64),\n ('isbidproduct_390L_approved_last', Boolean),\n ('isbidproduct_390L_rejected_count', Int64),\n ('isbidproduct_390L_rejected_last', Boolean),\n ('mainoccupationinc_437A_approved_max', Float64),\n ('mainoccupationinc_437A_approved_avg', Float64),\n ('mainoccupationinc_437A_approved_min', Float64),\n ('mainoccupationinc_437A_approved_std', String),\n ('mainoccupationinc_437A_approved_last', Float64),\n ('mainoccupationinc_437A_rejected_max', Float64),\n ('mainoccupationinc_437A_rejected_avg', Float64),\n ('mainoccupationinc_437A_rejected_min', Float64),\n ('mainoccupationinc_437A_rejected_std', Float64),\n ('mainoccupationinc_437A_rejected_last', Float64),\n ('mainoccupationinc_437A_last', Float64),\n ('maxdpdtolerance_577P_approved_max', Float64),\n ('maxdpdtolerance_577P_approved_avg', Float64),\n ('maxdpdtolerance_577P_approved_min', Float64),\n ('maxdpdtolerance_577P_approved_std', String),\n ('maxdpdtolerance_577P_approved_sum', Float64),\n ('maxdpdtolerance_577P_approved_last', Float64),\n ('outstandingdebt_522A_approved_max', Float64),\n ('outstandingdebt_522A_approved_avg', Float64),\n ('outstandingdebt_522A_approved_min', Float64),\n ('outstandingdebt_522A_approved_std', String),\n ('outstandingdebt_522A_approved_last', Float64),\n ('outstandingdebt_522A_rejected_max', Float64),\n ('outstandingdebt_522A_rejected_avg', Float64),\n ('outstandingdebt_522A_rejected_min', Float64),\n ('outstandingdebt_522A_rejected_std', Float64),\n ('outstandingdebt_522A_rejected_last', Float64),\n ('outstandingdebt_522A_last', Float64),\n ('pmtnum_8L_approved_max', Float64),\n ('pmtnum_8L_approved_avg', Float64),\n ('pmtnum_8L_approved_min', Float64),\n ('pmtnum_8L_approved_std', String),\n ('pmtnum_8L_approved_last', Float64),\n ('pmtnum_8L_rejected_max', Float64),\n ('pmtnum_8L_rejected_avg', Float64),\n ('pmtnum_8L_rejected_min', Float64),\n ('pmtnum_8L_rejected_std', Float64),\n ('pmtnum_8L_rejected_last', Float64),\n ('pmtnum_8L_last', Float64),\n ('postype_4733339M_mode', String),\n ('postype_4733339M_last', String),\n ('profession_152M_mode', String),\n ('profession_152M_last', String),\n ('rejectreason_755M_approved_mode', String),\n ('rejectreason_755M_rejected_mode', String),\n ('rejectreason_755M_overall_mode', String),\n ('rejectreason_755M_last', String),\n ('rejectreasonclient_4145042M_approved_mode', String),\n ('rejectreasonclient_4145042M_rejected_mode', String),\n ('rejectreasonclient_4145042M_overall_mode', String),\n ('rejectreasonclient_4145042M_last', String),\n ('status_219L_mode', String),\n ('status_219L_last', String),\n ('tenor_203L_approved_max', Float64),\n ('tenor_203L_approved_avg', Float64),\n ('tenor_203L_approved_min', Float64),\n ('tenor_203L_approved_std', String),\n ('tenor_203L_approved_last', Float64),\n ('tenor_203L_rejected_max', Float64),\n ('tenor_203L_rejected_avg', Float64),\n ('tenor_203L_rejected_min', Float64),\n ('tenor_203L_rejected_std', Float64),\n ('tenor_203L_rejected_last', Float64),\n ('tenor_203L_last', Float64),\n ('cacccardblochreas_147M_mode', String),\n ('credacc_cards_status_52L_mode', String),\n ('diff_date_decision_vs_approval_last', Int64),\n ('diff_date_decision_vs_creation_last', Int64),\n ('has_applprev', Int32),\n ('annualeffectiverate_199L_mean', Float64),\n ('annualeffectiverate_63L_mean', Float64),\n ('credlmt_230A_mean', Float64),\n ('credlmt_935A_mean', Float64),\n ('debtoutstand_525A_mean', Float64),\n ('debtoverdue_47A_mean', Float64),\n ('dpdmax_139P_mean', Float64),\n ('dpdmax_757P_mean', Float64),\n ('instlamount_768A_mean', Float64),\n ('instlamount_852A_mean', Float64),\n ('monthlyinstlamount_332A_mean', Float64),\n ('monthlyinstlamount_674A_mean', Float64),\n ('nominalrate_281L_mean', Float64),\n ('nominalrate_498L_mean', Float64),\n ('numberofcontrsvalue_258L_mean', Float64),\n ('numberofcontrsvalue_358L_mean', Float64),\n ('numberofinstls_229L_mean', Float64),\n ('numberofinstls_320L_mean', Float64),\n ('numberofoutstandinstls_520L_mean', Float64),\n ('numberofoutstandinstls_59L_mean', Float64),\n ('numberofoverdueinstlmax_1039L_mean', Float64),\n ('numberofoverdueinstlmax_1151L_mean', Float64),\n ('numberofoverdueinstls_725L_mean', Float64),\n ('numberofoverdueinstls_834L_mean', Float64),\n ('outstandingamount_354A_mean', Float64),\n ('outstandingamount_362A_mean', Float64),\n ('overdueamount_31A_mean', Float64),\n ('overdueamount_659A_mean', Float64),\n ('overdueamountmax2_14A_mean', Float64),\n ('overdueamountmax2_398A_mean', Float64),\n ('overdueamountmax_155A_mean', Float64),\n ('overdueamountmax_35A_mean', Float64),\n ('periodicityofpmts_1102L_mean', Float64),\n ('periodicityofpmts_837L_mean', Float64),\n ('prolongationcount_1120L_mean', Float64),\n ('prolongationcount_599L_mean', Float64),\n ('residualamount_488A_mean', Float64),\n ('residualamount_856A_mean', Float64),\n ('totalamount_6A_mean', Float64),\n ('totalamount_996A_mean', Float64),\n ('totaldebtoverduevalue_178A_mean', Float64),\n ('totaldebtoverduevalue_718A_mean', Float64),\n ('totaloutstanddebtvalue_39A_mean', Float64),\n ('totaloutstanddebtvalue_668A_mean', Float64),\n ('diff_dateofcre_and_dateof_end_active_mean', Float64),\n ('diff_dateofcre_and_dateof_end_close_mean', Float64),\n ('diff_date_decision_dateofcredend_289D_mean', Float64),\n ('diff_date_decision_dateofcredend_353D_mean', Float64),\n ('diff_date_decision_dateofcredstart_181D_mean', Float64),\n ('diff_date_decision_dateofcredstart_739D_mean', Float64),\n ('diff_date_decision_dateofrealrepmt_138D_mean', Float64),\n ('diff_date_decision_lastupdate_1112D_mean', Float64),\n ('diff_date_decision_lastupdate_388D_mean', Float64),\n ('dpdmax_139P_max', Float64),\n ('dpdmax_757P_max', Float64),\n ('instlamount_768A_max', Float64),\n ('instlamount_852A_max', Float64),\n ('monthlyinstlamount_332A_max', Float64),\n ('monthlyinstlamount_674A_max', Float64),\n ('nominalrate_281L_max', Float64),\n ('nominalrate_498L_max', Float64),\n ('numberofcontrsvalue_258L_max', Float64),\n ('numberofcontrsvalue_358L_max', Float64),\n ('numberofinstls_229L_max', Float64),\n ('numberofinstls_320L_max', Float64),\n ('numberofoutstandinstls_520L_max', Float64),\n ('numberofoutstandinstls_59L_max', Float64),\n ('numberofoverdueinstlmax_1039L_max', Float64),\n ('numberofoverdueinstlmax_1151L_max', Float64),\n ('numberofoverdueinstls_725L_max', Float64),\n ('numberofoverdueinstls_834L_max', Float64),\n ('outstandingamount_354A_max', Float64),\n ('outstandingamount_362A_max', Float64),\n ('overdueamount_31A_max', Float64),\n ('overdueamount_659A_max', Float64),\n ('overdueamountmax2_14A_max', Float64),\n ('overdueamountmax2_398A_max', Float64),\n ('overdueamountmax_155A_max', Float64),\n ('overdueamountmax_35A_max', Float64),\n ('periodicityofpmts_1102L_max', Float64),\n ('periodicityofpmts_837L_max', Float64),\n ('prolongationcount_1120L_max', Float64),\n ('prolongationcount_599L_max', Float64),\n ('residualamount_488A_max', Float64),\n ('residualamount_856A_max', Float64),\n ('totalamount_6A_max', Float64),\n ('totalamount_996A_max', Float64),\n ('totaldebtoverduevalue_178A_max', Float64),\n ('totaldebtoverduevalue_718A_max', Float64),\n ('totaloutstanddebtvalue_39A_max', Float64),\n ('totaloutstanddebtvalue_668A_max', Float64),\n ('diff_dateofcre_and_dateof_end_active_max', Int64),\n ('diff_dateofcre_and_dateof_end_close_max', Int64),\n ('diff_date_decision_dateofcredend_289D_max', Int64),\n ('diff_date_decision_dateofcredend_353D_max', Int64),\n ('diff_date_decision_dateofcredstart_181D_max', Int64),\n ('diff_date_decision_dateofcredstart_739D_max', Int64),\n ('diff_date_decision_dateofrealrepmt_138D_max', Int64),\n ('diff_date_decision_lastupdate_1112D_max', Int64),\n ('diff_date_decision_lastupdate_388D_max', Int64),\n ('contractsum_5085717L_sum', Float64),\n ('debtoutstand_525A_sum', Float64),\n ('debtoverdue_47A_sum', Float64),\n ('instlamount_768A_sum', Float64),\n ('instlamount_852A_sum', Float64),\n ('monthlyinstlamount_332A_sum', Float64),\n ('monthlyinstlamount_674A_sum', Float64),\n ('outstandingamount_354A_sum', Float64),\n ('outstandingamount_362A_sum', Float64),\n ('overdueamount_31A_sum', Float64),\n ('overdueamount_659A_sum', Float64),\n ('overdueamountmax2_14A_sum', Float64),\n ('overdueamountmax2_398A_sum', Float64),\n ('overdueamountmax_155A_sum', Float64),\n ('overdueamountmax_35A_sum', Float64),\n ('periodicityofpmts_1102L_sum', Float64),\n ('periodicityofpmts_837L_sum', Float64),\n ('prolongationcount_1120L_sum', Float64),\n ('prolongationcount_599L_sum', Float64),\n ('residualamount_488A_sum', Float64),\n ('residualamount_856A_sum', Float64),\n ('totalamount_6A_sum', Float64),\n ('totalamount_996A_sum', Float64),\n ('totaldebtoverduevalue_178A_sum', Float64),\n ('totaldebtoverduevalue_718A_sum', Float64),\n ('totaloutstanddebtvalue_39A_sum', Float64),\n ('totaloutstanddebtvalue_668A_sum', Float64),\n ('classificationofcontr_13M_mode', String),\n ('classificationofcontr_400M_mode', String),\n ('contractst_545M_mode', String),\n ('contractst_964M_mode', String),\n ('description_351M_mode', String),\n ('dpdmaxdatemonth_442T_mode', Float64),\n ('dpdmaxdatemonth_89T_mode', Float64),\n ('dpdmaxdateyear_596T_mode', Float64),\n ('dpdmaxdateyear_896T_mode', Float64),\n ('financialinstitution_382M_mode', String),\n ('financialinstitution_591M_mode', String),\n ('overdueamountmaxdatemonth_284T_mode', Float64),\n ('overdueamountmaxdatemonth_365T_mode', Float64),\n ('overdueamountmaxdateyear_2T_mode', Float64),\n ('overdueamountmaxdateyear_994T_mode', Float64),\n ('purposeofcred_426M_mode', String),\n ('purposeofcred_874M_mode', String),\n ('subjectrole_182M_mode', String),\n ('subjectrole_93M_mode', String),\n ('case_id_count', Int64),\n ('overdue_ratio_active_mean', Float64),\n ('overdue_ratio_active_max', Float64),\n ('overdue_ratio_active_sum', Float64),\n ('overdue_ratio_closed_mean', Float64),\n ('overdue_ratio_closed_max', Float64),\n ('overdue_ratio_closed_sum', Float64),\n ('overdue_ratio_max_active_mean', Float64),\n ('overdue_ratio_max_closed_mean', Float64),\n ('dpd_ratio_active_mean', Float64),\n ('dpd_ratio_active_max', Float64),\n ('dpd_ratio_closed_mean', Float64),\n ('dpd_ratio_closed_max', Float64),\n ('utilization_ratio_active_mean', Float64),\n ('utilization_ratio_active_max', Float64),\n ('utilization_ratio_closed_mean', Float64),\n ('utilization_ratio_closed_max', Float64),\n ('debt_to_limit_ratio_mean', Float64),\n ('debt_to_limit_ratio_sum', Float64),\n ('dpd_normalized_active_mean', Float64),\n ('dpd_normalized_active_max', Float64),\n ('dpd_normalized_closed_mean', Float64),\n ('dpd_normalized_closed_max', Float64),\n ('overdue_normalized_active_mean', Float64),\n ('overdue_normalized_active_max', Float64),\n ('overdue_normalized_closed_mean', Float64),\n ('overdue_normalized_closed_max', Float64),\n ('prolongation_normalized_active_mean', Float64),\n ('prolongation_normalized_active_max', Float64),\n ('prolongation_normalized_closed_mean', Float64),\n ('prolongation_normalized_closed_max', Float64),\n ('overall_risk_score_active_mean', Float64),\n ('overall_risk_score_active_max', Float64),\n ('overall_risk_score_closed_mean', Float64),\n ('overall_risk_score_closed_max', Float64),\n ('payment_regularity_active_mean', Float64),\n ('payment_regularity_active_max', Float64),\n ('payment_regularity_closed_mean', Float64),\n ('payment_regularity_closed_max', Float64),\n ('payment_consistency_active_mean', Float64),\n ('payment_consistency_active_max', Float64),\n ('payment_consistency_closed_mean', Float64),\n ('payment_consistency_closed_max', Float64),\n ('payment_behavior_score_active_mean', Float64),\n ('payment_behavior_score_active_max', Float64),\n ('payment_behavior_score_closed_mean', Float64),\n ('payment_behavior_score_closed_max', Float64),\n ('completion_rate_closed_mean', Float64),\n ('completion_rate_closed_max', Float64),\n ('progress_rate_active_mean', Float64),\n ('progress_rate_active_max', Float64),\n ('stability_indicator_active_mean', Float64),\n ('stability_indicator_active_max', Float64),\n ('stability_indicator_closed_mean', Float64),\n ('stability_indicator_closed_max', Float64),\n ('contract_stability_score_active_mean', Float64),\n ('contract_stability_score_active_max', Float64),\n ('contract_stability_score_closed_mean', Float64),\n ('contract_stability_score_closed_max', Float64),\n ('recent_update_active', Boolean),\n ('recent_update_closed', Boolean),\n ('recent_overdue_trend_active', Float64),\n ('recent_overdue_trend_closed', Float64),\n ('outstanding_trend_active_mean', Float64),\n ('outstanding_trend_active_max', Float64),\n ('outstanding_trend_closed_mean', Float64),\n ('outstanding_trend_closed_max', Float64),\n ('total_exposure_sum', Float64),\n ('total_exposure_mean', Float64),\n ('risk_weighted_exposure_active_mean', Float64),\n ('risk_weighted_exposure_closed_mean', Float64),\n ('active_closed_ratio_mean', Float64),\n ('active_closed_ratio_sum', Float64),\n ('avg_contract_size_active_mean', Float64),\n ('avg_contract_size_closed_mean', Float64),\n ('deterioration_signal_active', Boolean),\n ('deterioration_signal_closed', Boolean),\n ('high_risk_flag_active', Boolean),\n ('payment_stress_flag_active', Boolean),\n ('has_credit_bureau', Int32),\n ('tax_record_count', Int64),\n ('tax_amount_sum', Float64),\n ('tax_amount_avg', Float64),\n ('tax_amount_min', Float64),\n ('tax_amount_max', Float64),\n ('tax_amount_std', Float64),\n ('employer_nunique', Int64),\n ('tax_date_first', String),\n ('tax_date_last', String),\n ('diff_tax_date_date_decision_max', Int64),\n ('diff_tax_date_date_decision_min', Int64),\n ('diff_tax_date_date_decision_std', Float64),\n ('diff_tax_date_date_decision_mean', Float64),\n ('diff_tax_date_date_decision_last', Int64),\n ('tax_date_duration_days', Int64),\n ('tax_interval_avg_days', Float64),\n ('tax_interval_ratio', Float64),\n ('tax_frequency_per_day', Float64),\n ('has_tax', Int32),\n ('person_count', Int64),\n ('age_mean', Float64),\n ('age_min', Int64),\n ('age_max', Int64),\n ('contaddr_district_15M_nunique', Int64),\n ('contaddr_district_15M_mode', String),\n ('education_927M_mode', String),\n ('incometype_1044T_nunique', Int64),\n ('incometype_1044T_mode', String),\n ('language1_981M_nunique', Int64),\n ('language1_981M_mode', String),\n ('mainoccupationinc_384A_mean', Float64),\n ('mainoccupationinc_384A_sum', Float64),\n ('mainoccupationinc_384A_max', Float64),\n ('registaddr_district_1083M_mode', String),\n ('role_1084L_mode', String),\n ('sex_738L_nunique', Int64),\n ('sex_738L_mode', String),\n ('type_25L_nunique', Int64),\n ('type_25L_mode', String),\n ('addres_district_368M_mode', String),\n ('conts_role_79M_mode', String),\n ('has_person', Int32),\n ('amount_sum', Float64),\n ('amount_mean', Float64),\n ('count', Int64),\n ('has_deposit', Int32),\n ('actualdpdtolerance_344P', Float64),\n ('amtinstpaidbefduel24m_4187115A', String),\n ('annuity_780A', Float64),\n ('annuitynextmonth_57A', Float64),\n ('applicationcnt_361L', Float64),\n ('applications30d_658L', Float64),\n ('applicationscnt_1086L', Float64),\n ('applicationscnt_464L', Float64),\n ('applicationscnt_629L', Float64),\n ('applicationscnt_867L', Float64),\n ('avgdbddpdlast24m_3658932P', Float64),\n ('avgdbddpdlast3m_4187120P', String),\n ('avgdbdtollast24m_4525197P', String),\n ('avgdpdtolclosure24_3658938P', Float64),\n ('avginstallast24m_3658937A', Float64),\n ('avglnamtstart24m_4525187A', String),\n ('avgmaxdpdlast9m_3716943P', Float64),\n ('avgoutstandbalancel6m_4187114A', String),\n ('avgpmtlast12m_4525200A', String),\n ('bankacctype_710L', String),\n ('cardtype_51L', String),\n ('clientscnt12m_3712952L', Float64),\n ('clientscnt3m_3712950L', Float64),\n ('clientscnt6m_3712949L', Float64),\n ('clientscnt_100L', Float64),\n ('clientscnt_1022L', Float64),\n ('clientscnt_1071L', Float64),\n ('clientscnt_1130L', Float64),\n ('clientscnt_136L', Float64),\n ('clientscnt_157L', Float64),\n ('clientscnt_257L', Float64),\n ('clientscnt_304L', Float64),\n ('clientscnt_360L', Float64),\n ('clientscnt_493L', Float64),\n ('clientscnt_533L', Float64),\n ('clientscnt_887L', Float64),\n ('clientscnt_946L', Float64),\n ('cntincpaycont9m_3716944L', Float64),\n ('cntpmts24_3658933L', Float64),\n ('commnoinclast6m_3546845L', Float64),\n ('credamount_770A', Float64),\n ('credtype_322L', String),\n ('currdebt_22A', Float64),\n ('currdebtcredtyperange_828A', Float64),\n ('daysoverduetolerancedd_3976961L', Float64),\n ('deferredmnthsnum_166L', Float64),\n ('disbursedcredamount_1113A', Float64),\n ('disbursementtype_67L', String),\n ('downpmt_116A', Float64),\n ('eir_270L', Float64),\n ('homephncnt_628L', Float64),\n ('inittransactionamount_650A', Float64),\n ('inittransactioncode_186L', String),\n ('interestrate_311L', Float64),\n ('interestrategrace_34L', Float64),\n ('lastapprcommoditycat_1041M', String),\n ('lastapprcommoditytypec_5251766M', String),\n ('lastapprcredamount_781A', Float64),\n ('lastcancelreason_561M', String),\n ('lastdependentsnum_448L', String),\n ('lastotherinc_902A', String),\n ('lastotherlnsexpense_631A', String),\n ('lastrejectcommoditycat_161M', String),\n ('lastrejectcommodtypec_5251769M', String),\n ('lastrejectcredamount_222A', Float64),\n ('lastrejectreason_759M', String),\n ('lastrejectreasonclient_4145040M', String),\n ('lastst_736L', String),\n ('maininc_215A', Float64),\n ('maxannuity_159A', Float64),\n ('maxannuity_4075009A', String),\n ('maxdbddpdlast1m_3658939P', Float64),\n ('maxdbddpdtollast12m_3658940P', Float64),\n ('maxdbddpdtollast6m_4187119P', String),\n ('maxdebt4_972A', Float64),\n ('maxdpdfrom6mto36m_3546853P', Float64),\n ('maxdpdinstldate_3546855D', String),\n ('maxdpdinstlnum_3546846P', Float64),\n ('maxdpdlast12m_727P', Float64),\n ('maxdpdlast24m_143P', Float64),\n ('maxdpdlast3m_392P', Float64),\n ('maxdpdlast6m_474P', Float64),\n ('maxdpdlast9m_1059P', Float64),\n ('maxdpdtolerance_374P', Float64),\n ('maxinstallast24m_3658928A', Float64),\n ('maxlnamtstart6m_4525199A', String),\n ('maxoutstandbalancel12m_4187113A', String),\n ('maxpmtlast3m_4525190A', String),\n ('mindbddpdlast24m_3658935P', Float64),\n ('mindbdtollast24m_4525191P', String),\n ('mobilephncnt_593L', Float64),\n ('monthsannuity_845L', Float64),\n ('numactivecreds_622L', Float64),\n ('numactivecredschannel_414L', Float64),\n ('numactiverelcontr_750L', Float64),\n ('numcontrs3months_479L', Float64),\n ('numincomingpmts_3546848L', Float64),\n ('numinstlallpaidearly3d_817L', Float64),\n ('numinstls_657L', Float64),\n ('numinstlsallpaid_934L', Float64),\n ('numinstlswithdpd10_728L', Float64),\n ('numinstlswithdpd5_4187116L', String),\n ('numinstlswithoutdpd_562L', Float64),\n ('numinstmatpaidtearly2d_4499204L', String),\n ('numinstpaid_4499208L', String),\n ('numinstpaidearly3d_3546850L', Float64),\n ('numinstpaidearly3dest_4493216L', String),\n ('numinstpaidearly5d_1087L', Float64),\n ('numinstpaidearly5dest_4493211L', String),\n ('numinstpaidearly5dobd_4499205L', String),\n ('numinstpaidearly_338L', Float64),\n ('numinstpaidearlyest_4493214L', String),\n ('numinstpaidlastcontr_4325080L', String),\n ('numinstpaidlate1d_3546852L', Float64),\n ('numinstregularpaid_973L', Float64),\n ('numinstregularpaidest_4493210L', String),\n ('numinsttopaygr_769L', Float64),\n ('numinsttopaygrest_4493213L', String),\n ('numinstunpaidmax_3546851L', Float64),\n ('numinstunpaidmaxest_4493212L', String),\n ('numnotactivated_1143L', Float64),\n ('numpmtchanneldd_318L', Float64),\n ('numrejects9m_859L', Float64),\n ('opencred_647L', Boolean),\n ('paytype1st_925L', String),\n ('paytype_783L', String),\n ('payvacationpostpone_4187118D', String),\n ('pctinstlsallpaidearl3d_427L', Float64),\n ('pctinstlsallpaidlat10d_839L', Float64),\n ('pctinstlsallpaidlate1d_3546856L', Float64),\n ('pctinstlsallpaidlate4d_3546849L', Float64),\n ('pctinstlsallpaidlate6d_3546844L', Float64),\n ('pmtnum_254L', Float64),\n ('posfpd10lastmonth_333P', Float64),\n ('posfpd30lastmonth_3976960P', Float64),\n ('posfstqpd30lastmonth_3976962P', Float64),\n ('previouscontdistrict_112M', String),\n ('price_1097A', Float64),\n ('sellerplacecnt_915L', Float64),\n ('sellerplacescnt_216L', Float64),\n ('sumoutstandtotal_3546847A', Float64),\n ('sumoutstandtotalest_4493215A', String),\n ('totaldebt_9A', Float64),\n ('totalsettled_863A', Float64),\n ('totinstallast1m_4525188A', String),\n ('twobodfilling_608L', String),\n ('typesuite_864L', String),\n ('diff_datelastunpaid_3546854D_to_decision', Int64),\n ('diff_dtlastpmtallstes_4499206D_to_decision', String),\n ('diff_lastapprdate_640D_to_decision', Int64),\n ('diff_lastdelinqdate_224D_to_decision', Int64),\n ('diff_lastrejectdate_50D_to_decision', Int64),\n ('diff_firstdatedue_to_lastpmt', String),\n ('equalitydataagreement_891L', Int64),\n ('equalityempfrom_62L', Int64),\n ('isbidproduct_1095L', Int64),\n ('isbidproductrequest_292L', Int64),\n ('isdebitcard_729L', Int64),\n ('mastercontrelectronic_519L', Int64),\n ('mastercontrexist_109L', Int64),\n ('has_static', Int32),\n ('education', String),\n ('marital_status', String),\n ('pmtaverage', Float64),\n ('pmtcount', Float64),\n ('contractssum_5085716L', String),\n ('days120_123L', Float64),\n ('days180_256L', Float64),\n ('days30_165L', Float64),\n ('days360_512L', Float64),\n ('days90_310L', Float64),\n ('description_5085714M', String),\n ('firstquarter_103L', Float64),\n ('for3years_128L', Float64),\n ('for3years_504L', Float64),\n ('for3years_584L', Float64),\n ('formonth_118L', Float64),\n ('formonth_206L', Float64),\n ('formonth_535L', Float64),\n ('forquarter_1017L', Float64),\n ('forquarter_462L', Float64),\n ('forquarter_634L', Float64),\n ('fortoday_1092L', Float64),\n ('forweek_1077L', Float64),\n ('forweek_528L', Float64),\n ('forweek_601L', Float64),\n ('foryear_618L', Float64),\n ('foryear_818L', Float64),\n ('foryear_850L', Float64),\n ('fourthquarter_440L', Float64),\n ('numberofqueries_373L', Float64),\n ('pmtscount_423L', Float64),\n ('pmtssum_45A', Float64),\n ('requesttype_4525192L', String),\n ('riskassesment_302T', String),\n ('riskassesment_940T', Float64),\n ('secondquarter_766L', Float64),\n ('thirdquarter_1082L', Float64),\n ('age_estimate', Int64),\n ('processing_days', Int64),\n ('credit_activity_ratio', Float64),\n ('midterm_credit_ratio', Float64),\n ('rejection_density', Float64),\n ('shortterm_rejection_density', Float64),\n ('tax_burden', String),\n ('tax_compliance', Float64),\n ('response_date_date_decision', Int64),\n ('has_static_cb', Int32),\n ('amtdebitincoming_4809443A', Float64),\n ('amtdebitoutgoing_4809440A', Float64),\n ('amtdepositbalance_4809441A', Float64),\n ('amtdepositincoming_4809444A', Float64),\n ('amtdepositoutgoing_4809442A', Float64),\n ('num_group1', Int64),\n ('has_other', Int32)])"},"metadata":{}}],"execution_count":11},{"cell_type":"code","source":"cols_to_drop = [\n 'actualdpd_943P_approved_std', 'annuity_853A_approved_std', 'credacc_credlmt_575A_approved_std',\n 'credamount_590A_approved_std', 'currdebt_94A_approved_std', 'downpmt_134A_approved_std',\n 'rate_downpmt_and_credamount_approved_std', 'employedfrom_creationdate_approved_std',\n 'diff_firstnonzeroinstldate_creationdate_approved_std', 'mainoccupationinc_437A_approved_std',\n 'maxdpdtolerance_577P_approved_std', 'outstandingdebt_522A_approved_std',\n 'tenor_203L_approved_std', 'recent_update_active', 'deterioration_signal_active',\n 'deterioration_signal_closed', 'high_risk_flag_active', 'payment_stress_flag_active',\n 'tax_date_first', 'tax_date_last', 'lastapprcommoditycat_1041M',\n 'lastapprcommoditytypec_5251766M', 'lastcancelreason_561M', 'lastdependentsnum_448L',\n 'lastotherinc_902A', 'lastotherlnsexpense_631A', 'lastrejectcommoditycat_161M',\n 'lastrejectcommodtypec_5251769M', 'lastrejectreason_759M', 'lastrejectreasonclient_4145040M',\n 'lastst_736L', 'maxdpdinstldate_3546855D', 'opencred_647L', 'payvacationpostpone_4187118D',\n 'diff_dtlastpmtallstes_4499206D_to_decision', 'diff_firstdatedue_to_lastpmt','pmtnum_8L_approved_std','district_544M_last','district_544M_mode'\n 'contractssum_5085716L','profession_152M_mode','profession_152M_last',\"district_544M_mode\",\n \"district_544M_last\",\n \"pmtnum_8L_approved_std\",\n \"profession_152M_mode\",\n \"profession_152M_last\",\n \"classificationofcontr_400M_mode\",\n \"contractst_964M_mode\",\n \"financialinstitution_382M_mode\",\n \"financialinstitution_591M_mode\",\n \"contaddr_district_15M_mode\",\n \"registaddr_district_1083M_mode\",\n \"addres_district_368M_mode\",\n \"previouscontdistrict_112M\",\n \"contractssum_5085716L\",'tax_burden'\n]\ncols_to_convert_to_numeric = [\n 'amtinstpaidbefduel24m_4187115A', 'avgdbdtollast24m_4525197P', 'avgdbddpdlast3m_4187120P',\n 'avglnamtstart24m_4525187A', 'avgoutstandbalancel6m_4187114A', 'avgpmtlast12m_4525200A',\n 'maxannuity_159A', 'maxdbddpdtollast6m_4187119P', 'maxoutstandbalancel12m_4187113A',\n 'maxpmtlast3m_4525190A', 'mindbdtollast24m_4525191P', 'numinstlswithdpd5_4187116L',\n 'numinstlswithoutdpd_562L', 'numinstmatpaidtearly2d_4499204L', 'numinstpaid_4499208L',\n 'numinstpaidearly3d_3546850L', 'numinstpaidearly3dest_4493216L', 'numinstpaidearly5d_1087L',\n 'numinstpaidearly5dest_4493211L', 'numinstpaidearly5dobd_4499205L', 'numinstpaidearly_338L',\n 'numinstpaidearlyest_4493214L', 'numinstpaidlastcontr_4325080L', 'numinstpaidlate1d_3546852L',\n 'numinstregularpaid_973L', 'numinstregularpaidest_4493210L', 'numinsttopaygr_769L',\n 'numinsttopaygrest_4493213L', 'numinstunpaidmax_3546851L', 'numinstunpaidmaxest_4493212L',\n 'numnotactivated_1143L', 'numpmtchanneldd_318L', 'numrejects9m_859L',\n 'sumoutstandtotalest_4493215A', 'totinstallast1m_4525188A','childnum_21L_approved_avg','maxannuity_4075009A','maxlnamtstart6m_4525199A','classificationofcontr_400M_mode'\n]\n# 1. Drop cột\nexisting_drop_cols = [col for col in cols_to_drop if col in train_merged.columns]\ntrain_merged = train_merged.drop(existing_drop_cols)\n\n# 2. Ép kiểu về số nếu là string\nexisting_convert_cols = [col for col in cols_to_convert_to_numeric if col in train_merged.columns]\ntrain_merged = train_merged.with_columns([\n pl.col(col).cast(pl.Float64, strict=False).alias(col)\n for col in existing_convert_cols\n])","metadata":{"_uuid":"df099caa-3f4c-48c2-aa72-60ef958b144c","_cell_guid":"a7931335-476f-417f-afa8-e1927a91bd57","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:44.947683Z","iopub.execute_input":"2025-06-23T20:39:44.947932Z","iopub.status.idle":"2025-06-23T20:39:45.015761Z","shell.execute_reply.started":"2025-06-23T20:39:44.947908Z","shell.execute_reply":"2025-06-23T20:39:45.011885Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":12},{"cell_type":"markdown","source":"## Null handle","metadata":{"_uuid":"8ee1ca5d-cd91-401d-87af-68a24974d8c6","_cell_guid":"aae73e84-5ace-4c45-b17d-e09e547aa09c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Danh sách các cột has_* cần fill null = 0\nhas_columns = [\n 'has_applprev', 'has_credit_bureau', 'has_tax',\n 'has_person', 'has_deposit', 'has_static',\n 'has_static_cb', 'has_other'\n]\n\n# Fill null các cột này bằng 0, inplace (không sinh thêm cột)\ntrain_merged = train_merged.with_columns([\n pl.col(col).fill_null(0) for col in has_columns\n])\n\n# Xoá cột num_group1 nếu có tồn tại trong dataframe\nif 'num_group1' in train_merged.columns:\n train_merged = train_merged.drop('num_group1')","metadata":{"_uuid":"c079173c-3ef6-47b4-b67a-238605528410","_cell_guid":"37d2a2f2-afe0-40a4-8034-bf82072a58cf","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:45.017635Z","iopub.execute_input":"2025-06-23T20:39:45.017882Z","iopub.status.idle":"2025-06-23T20:39:45.035676Z","shell.execute_reply.started":"2025-06-23T20:39:45.017857Z","shell.execute_reply":"2025-06-23T20:39:45.031438Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":13},{"cell_type":"code","source":"train_merged","metadata":{"_uuid":"f0521fa7-fbcd-487e-805e-e9e4063eeba7","_cell_guid":"8a244b0a-e9e3-4b94-82cc-eb9f20cf322a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:45.038242Z","iopub.execute_input":"2025-06-23T20:39:45.038473Z","iopub.status.idle":"2025-06-23T20:39:45.061642Z","shell.execute_reply.started":"2025-06-23T20:39:45.038449Z","shell.execute_reply":"2025-06-23T20:39:45.058633Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"shape: (1_526_659, 633)\n┌─────────┬────────┬──────────┬────────┬───┬──────────────┬──────────────┬─────────────┬───────────┐\n│ case_id ┆ MONTH ┆ WEEK_NUM ┆ target ┆ … ┆ amtdepositba ┆ amtdepositin ┆ amtdeposito ┆ has_other │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ lance_480944 ┆ coming_48094 ┆ utgoing_480 ┆ --- │\n│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ 1A ┆ 44A ┆ 9442A ┆ i32 │\n│ ┆ ┆ ┆ ┆ ┆ --- ┆ --- ┆ --- ┆ │\n│ ┆ ┆ ┆ ┆ ┆ f64 ┆ f64 ┆ f64 ┆ │\n╞═════════╪════════╪══════════╪════════╪═══╪══════════════╪══════════════╪═════════════╪═══════════╡\n│ 0 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n│ 1 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n│ 2 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n│ 3 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n│ 4 ┆ 201901 ┆ 0 ┆ 1 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n│ 2703450 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 1 │\n│ 2703451 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 1 │\n│ 2703452 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n│ 2703453 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 822.0 ┆ 0.0 ┆ 6.8 ┆ 1 │\n│ 2703454 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ null ┆ null ┆ null ┆ 0 │\n└─────────┴────────┴──────────┴────────┴───┴──────────────┴──────────────┴─────────────┴───────────┘","text/html":"
\nshape: (1_526_659, 633)
case_idMONTHWEEK_NUMtargetnum_applicationsnum_approvedrate_approval_applicationactualdpd_943P_approved_maxactualdpd_943P_approved_avgactualdpd_943P_approved_minactualdpd_943P_approved_lastannuity_853A_approved_maxannuity_853A_approved_avgannuity_853A_approved_minannuity_853A_approved_lastannuity_853A_rejected_maxannuity_853A_rejected_avgannuity_853A_rejected_minannuity_853A_rejected_stdannuity_853A_rejected_lastannuity_853A_lastapprovaldate_319D_durationapprovaldate_319D_intervalbyoccupationinc_3656910L_maxbyoccupationinc_3656910L_avgbyoccupationinc_3656910L_minbyoccupationinc_3656910L_stdbyoccupationinc_3656910L_lastcancelreason_3545846M_approved_modecancelreason_3545846M_rejected_modecancelreason_3545846M_overall_modecancelreason_3545846M_lastchildnum_21L_approved_avgchildnum_21L_rejected_avgchildnum_21L_lastcreationdate_885D_duration_allcreationdate_885D_interval_approved…formonth_118Lformonth_206Lformonth_535Lforquarter_1017Lforquarter_462Lforquarter_634Lfortoday_1092Lforweek_1077Lforweek_528Lforweek_601Lforyear_618Lforyear_818Lforyear_850Lfourthquarter_440Lnumberofqueries_373Lpmtscount_423Lpmtssum_45Arequesttype_4525192Lriskassesment_302Triskassesment_940Tsecondquarter_766Lthirdquarter_1082Lage_estimateprocessing_dayscredit_activity_ratiomidterm_credit_ratiorejection_densityshortterm_rejection_densitytax_complianceresponse_date_date_decisionhas_static_cbamtdebitincoming_4809443Aamtdebitoutgoing_4809440Aamtdepositbalance_4809441Aamtdepositincoming_4809444Aamtdepositoutgoing_4809442Ahas_other
i64i64i64i64i64i64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64i64f64f64f64f64f64f64strstrstrstrf64f64f64i64f64…f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64strstrf64f64f64i64i64f64f64f64f64f64i64i32f64f64f64f64f64i32
020190100nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull…nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull0nullnullnullnullnull0
120190100nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull…nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull0nullnullnullnullnull0
220190100200.0nullnullnullnullnullnullnullnull1682.41161.3640.2736.9466871682.41682.4null0.0nullnullnullnullnullnull"a55475b1""a55475b1""a55475b1"null0.00.000.0…nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull0nullnullnullnullnull0
320190100100.0nullnullnullnullnullnullnullnull6140.06140.06140.0null6140.06140.0null0.0nullnullnullnullnullnull"P94_109_143""P94_109_143""P94_109_143"nullnullnull00.0…nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull0nullnullnullnullnull0
420190101100.0nullnullnullnullnullnullnullnull2556.62556.62556.6null2556.62556.6null0.0nullnullnullnullnullnull"P24_27_36""P24_27_36""P24_27_36"nullnullnull00.0…nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull0nullnullnullnullnull0
………………………………………………………………………………………………………………………………………………………………………………………………………
270345020201091013110.8461540.00.00.00.04965.22528.0000180.02102.230875.030875.030875.0null30875.02102.24502450.21.00.50.00.707107null"a55475b1""P94_109_143""a55475b1""a55475b1"0.333333nullnull4502450.2…nullnullnullnullnullnullnullnullnullnullnullnullnull1.00.0nullnullnullnullnull1.01.05810120.00.0nullnull1.01410.00.00.00.00.01
2703451202010910661.00.00.00.00.012809.26472.8666670.06191.6nullnullnullnullnull6191.61536307.210340.010340.010340.0nullnull"a55475b1"null"a55475b1""a55475b1"0.0nullnull1536307.2…nullnullnullnullnullnullnullnullnullnullnullnullnull0.00.0nullnullnullnullnull1.02.05556050.00.0nullnull1.014127500.027477.60.00.00.01
2703452202010910320.6666670.00.00.00.03243.40012372.200051501.01501.09048.09048.09048.0null9048.09048.0337337.0nullnullnullnullnull"a55475b1""P180_60_137""a55475b1""P180_60_137"nullnullnull705337.0…nullnullnullnullnullnullnullnullnullnullnullnullnull1.03.0nullnullnullnullnull0.04.0nullnull0.00.666667nullnullnull141nullnullnullnullnull0
2703453202010910980.8888890.00.00.00.05981.41962.1250020.02827.2416.2416.2416.2null416.22827.22363337.57142933059.033059.033059.0nullnull"a55475b1""a55475b1""a55475b1""a55475b1"0.00.0null2363337.571429…nullnullnullnullnullnullnullnullnullnullnullnullnull3.04.0nullnullnullnullnull2.01.05846280.250.5nullnull1.16666712113454.013333.4822.00.06.81
2703454202010910221.00.00.00.00.06726.64856.72986.82986.8nullnullnullnullnull2986.8325325.0nullnullnullnullnull"a55475b1"null"a55475b1""a55475b1"nullnullnull325325.0…nullnullnullnullnullnullnullnullnullnullnullnullnull0.01.0nullnullnullnullnull2.01.05273630.01.0nullnull1.0141nullnullnullnullnull0
"},"metadata":{}}],"execution_count":14},{"cell_type":"code","source":"import polars as pl\n\n# Xác định các kiểu dữ liệu số của polars\nnumeric_types = {pl.Int8, pl.Int16, pl.Int32, pl.Int64,\n pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,\n pl.Float32, pl.Float64}\n\n# Chọn các cột kiểu số\nnumeric_cols = [col for col, dtype in zip(train_merged.columns, train_merged.schema.values())\n if dtype in numeric_types]\n\n\n\n# Nhóm 1: Nếu giá trị âm → set về 0\ndpd_cols = [\"dpdmax_757P_mean\", \"dpdmax_757P_max\"]\n\ntrain_merged = train_merged.with_columns([\n pl.when(pl.col(col) < 0)\n .then(0)\n .otherwise(pl.col(col))\n .alias(col)\n for col in dpd_cols\n])\n\n# Nhóm 2: Nếu giá trị âm → lấy trị tuyệt đối (abs)\ndate_diff_cols = [\n \"diff_datelastunpaid_3546854D_to_decision\",\n \"diff_lastapprdate_640D_to_decision\",\n \"diff_lastdelinqdate_224D_to_decision\",\n \"diff_lastrejectdate_50D_to_decision\"\n]\n\ntrain_merged = train_merged.with_columns([\n pl.col(col).abs().alias(col) for col in date_diff_cols\n])\n\n# Tạo bảng tổng hợp: null count, zero count, negative count\nsummary = pl.DataFrame({\n \"column\": numeric_cols,\n \"null_count\": [train_merged[col].null_count() for col in numeric_cols],\n \"zero_count\": [(train_merged[col] == 0).sum() for col in numeric_cols],\n \"negative_count\": [(train_merged[col] < 0).sum() for col in numeric_cols],\n})","metadata":{"_uuid":"f238534d-7e83-4c27-a3ab-c82c0e75a39c","_cell_guid":"830fd6db-391c-4c5b-b309-05a750600947","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:45.063711Z","iopub.execute_input":"2025-06-23T20:39:45.065107Z","iopub.status.idle":"2025-06-23T20:39:46.763378Z","shell.execute_reply.started":"2025-06-23T20:39:45.065083Z","shell.execute_reply":"2025-06-23T20:39:46.758076Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":15},{"cell_type":"code","source":"summary","metadata":{"_uuid":"a7133281-9a66-4c7c-8691-7fbaff6f9fe3","_cell_guid":"b9991bee-d8e1-496e-b714-a1132388a823","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:46.764086Z","iopub.execute_input":"2025-06-23T20:39:46.764291Z","iopub.status.idle":"2025-06-23T20:39:46.777651Z","shell.execute_reply.started":"2025-06-23T20:39:46.764269Z","shell.execute_reply":"2025-06-23T20:39:46.773291Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"shape: (576, 4)\n┌─────────────────────────────┬────────────┬────────────┬────────────────┐\n│ column ┆ null_count ┆ zero_count ┆ negative_count │\n│ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ i64 ┆ i64 ┆ i64 │\n╞═════════════════════════════╪════════════╪════════════╪════════════════╡\n│ case_id ┆ 0 ┆ 1 ┆ 0 │\n│ MONTH ┆ 0 ┆ 0 ┆ 0 │\n│ WEEK_NUM ┆ 0 ┆ 16735 ┆ 0 │\n│ target ┆ 0 ┆ 1478665 ┆ 0 │\n│ num_applications ┆ 305137 ┆ 0 ┆ 0 │\n│ … ┆ … ┆ … ┆ … │\n│ amtdebitoutgoing_4809440A ┆ 1475550 ┆ 27286 ┆ 0 │\n│ amtdepositbalance_4809441A ┆ 1475550 ┆ 32235 ┆ 1 │\n│ amtdepositincoming_4809444A ┆ 1475550 ┆ 45985 ┆ 0 │\n│ amtdepositoutgoing_4809442A ┆ 1475550 ┆ 22433 ┆ 0 │\n│ has_other ┆ 0 ┆ 1475550 ┆ 0 │\n└─────────────────────────────┴────────────┴────────────┴────────────────┘","text/html":"
\nshape: (576, 4)
columnnull_countzero_countnegative_count
stri64i64i64
"case_id"010
"MONTH"000
"WEEK_NUM"0167350
"target"014786650
"num_applications"30513700
…………
"amtdebitoutgoing_4809440A"1475550272860
"amtdepositbalance_4809441A"1475550322351
"amtdepositincoming_4809444A"1475550459850
"amtdepositoutgoing_4809442A"1475550224330
"has_other"014755500
"},"metadata":{}}],"execution_count":16},{"cell_type":"code","source":"# Định nghĩa kiểu số\nnumeric_types = {\n pl.Int8, pl.Int16, pl.Int32, pl.Int64,\n pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,\n pl.Float32, pl.Float64\n}\n\n# Lấy schema\nschema = train_merged.schema\n\n# Phân loại\nnumeric_cols = [col for col, dtype in schema.items() if dtype in numeric_types]\nstring_cols = [col for col, dtype in schema.items() if dtype == pl.Utf8]\n\n# In thống kê\nprint(f\"🧮 Số lượng biến số : {len(numeric_cols)}\")\nprint(f\"🔤 Số lượng biến chữ : {len(string_cols)}\")","metadata":{"_uuid":"1ac42b8e-8e7e-4932-937c-b886f7484f2b","_cell_guid":"5174ede9-428a-4d5c-9a66-bf8cc7006cd1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:46.780036Z","iopub.execute_input":"2025-06-23T20:39:46.78028Z","iopub.status.idle":"2025-06-23T20:39:46.7925Z","shell.execute_reply.started":"2025-06-23T20:39:46.780257Z","shell.execute_reply":"2025-06-23T20:39:46.78803Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"🧮 Số lượng biến số : 576\n🔤 Số lượng biến chữ : 54\n","output_type":"stream"}],"execution_count":17},{"cell_type":"code","source":"# Lọc các cột kiểu chuỗi\nstring_cols = [col for col, dtype in train_merged.schema.items() if dtype == pl.Utf8]\n\n# In số lượng giá trị unique của từng cột chuỗi\nfor col in string_cols:\n n_unique = train_merged.select(pl.col(col).n_unique()).item()\n print(f\"{col}: {n_unique} unique value(s)\")","metadata":{"_uuid":"6c592d18-178a-4a3e-8d32-57d268bef7bf","_cell_guid":"24c9c2d3-c270-4c70-8d23-d065239a135e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:46.793191Z","iopub.execute_input":"2025-06-23T20:39:46.793421Z","iopub.status.idle":"2025-06-23T20:39:48.289442Z","shell.execute_reply.started":"2025-06-23T20:39:46.7934Z","shell.execute_reply":"2025-06-23T20:39:48.283796Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"cancelreason_3545846M_approved_mode: 31 unique value(s)\ncancelreason_3545846M_rejected_mode: 68 unique value(s)\ncancelreason_3545846M_overall_mode: 67 unique value(s)\ncancelreason_3545846M_last: 75 unique value(s)\ncredtype_587L_mode: 4 unique value(s)\ncredtype_587L_last: 4 unique value(s)\neducation_1138M_mode: 7 unique value(s)\neducation_1138M_last: 7 unique value(s)\nfamilystate_726L_mode: 6 unique value(s)\nfamilystate_726L_last: 6 unique value(s)\ninittransactioncode_279L_mode: 4 unique value(s)\ninittransactioncode_279L_last: 4 unique value(s)\npostype_4733339M_mode: 10 unique value(s)\npostype_4733339M_last: 10 unique value(s)\nrejectreason_755M_approved_mode: 17 unique value(s)\nrejectreason_755M_rejected_mode: 19 unique value(s)\nrejectreason_755M_overall_mode: 19 unique value(s)\nrejectreason_755M_last: 19 unique value(s)\nrejectreasonclient_4145042M_approved_mode: 7 unique value(s)\nrejectreasonclient_4145042M_rejected_mode: 13 unique value(s)\nrejectreasonclient_4145042M_overall_mode: 12 unique value(s)\nrejectreasonclient_4145042M_last: 15 unique value(s)\nstatus_219L_mode: 11 unique value(s)\nstatus_219L_last: 12 unique value(s)\ncacccardblochreas_147M_mode: 9 unique value(s)\ncredacc_cards_status_52L_mode: 8 unique value(s)\nclassificationofcontr_13M_mode: 12 unique value(s)\ncontractst_545M_mode: 44 unique value(s)\ndescription_351M_mode: 13 unique value(s)\npurposeofcred_426M_mode: 19 unique value(s)\npurposeofcred_874M_mode: 24 unique value(s)\nsubjectrole_182M_mode: 9 unique value(s)\nsubjectrole_93M_mode: 10 unique value(s)\neducation_927M_mode: 6 unique value(s)\nincometype_1044T_mode: 9 unique value(s)\nlanguage1_981M_mode: 3 unique value(s)\nrole_1084L_mode: 4 unique value(s)\nsex_738L_mode: 3 unique value(s)\ntype_25L_mode: 8 unique value(s)\nconts_role_79M_mode: 11 unique value(s)\nbankacctype_710L: 2 unique value(s)\ncardtype_51L: 3 unique value(s)\ncredtype_322L: 4 unique value(s)\ndisbursementtype_67L: 4 unique value(s)\ninittransactioncode_186L: 4 unique value(s)\npaytype1st_925L: 2 unique value(s)\npaytype_783L: 2 unique value(s)\ntwobodfilling_608L: 3 unique value(s)\ntypesuite_864L: 2 unique value(s)\neducation: 6 unique value(s)\nmarital_status: 7 unique value(s)\ndescription_5085714M: 3 unique value(s)\nrequesttype_4525192L: 4 unique value(s)\nriskassesment_302T: 17 unique value(s)\n","output_type":"stream"}],"execution_count":18},{"cell_type":"code","source":"# Lọc cột kiểu chuỗi\nstring_cols = [col for col, dtype in train_merged.schema.items() if dtype == pl.Utf8]\n\n# Duyệt và lọc theo số lượng unique > 50\nfor col in string_cols:\n n_unique = train_merged.select(pl.col(col).n_unique()).item()\n if n_unique > 50:\n print(f\"{col}: {n_unique} unique values\")","metadata":{"_uuid":"5a493f0d-6f02-4dfa-b25a-891d579da72c","_cell_guid":"626770c8-2666-4eea-a44b-f6207cc66c7f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:48.291157Z","iopub.execute_input":"2025-06-23T20:39:48.291387Z","iopub.status.idle":"2025-06-23T20:39:49.701209Z","shell.execute_reply.started":"2025-06-23T20:39:48.291364Z","shell.execute_reply":"2025-06-23T20:39:49.695552Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"cancelreason_3545846M_rejected_mode: 68 unique values\ncancelreason_3545846M_overall_mode: 67 unique values\ncancelreason_3545846M_last: 75 unique values\n","output_type":"stream"}],"execution_count":19},{"cell_type":"code","source":"summary = pl.DataFrame({\n \"column\": numeric_cols,\n \"null_count\": [train_merged[col].null_count() for col in numeric_cols],\n \"zero_count\": [(train_merged[col] == 0).sum() for col in numeric_cols],\n \"negative_count\": [(train_merged[col] < 0).sum() for col in numeric_cols],\n})\nsummary","metadata":{"_uuid":"282e8c59-dd73-4635-b7d8-04bf5f1499db","_cell_guid":"ea41d47d-1657-4bc6-903b-d5789f3ad83c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:49.703107Z","iopub.execute_input":"2025-06-23T20:39:49.703337Z","iopub.status.idle":"2025-06-23T20:39:51.419172Z","shell.execute_reply.started":"2025-06-23T20:39:49.703314Z","shell.execute_reply":"2025-06-23T20:39:51.414922Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"shape: (576, 4)\n┌─────────────────────────────┬────────────┬────────────┬────────────────┐\n│ column ┆ null_count ┆ zero_count ┆ negative_count │\n│ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ i64 ┆ i64 ┆ i64 │\n╞═════════════════════════════╪════════════╪════════════╪════════════════╡\n│ case_id ┆ 0 ┆ 1 ┆ 0 │\n│ MONTH ┆ 0 ┆ 0 ┆ 0 │\n│ WEEK_NUM ┆ 0 ┆ 16735 ┆ 0 │\n│ target ┆ 0 ┆ 1478665 ┆ 0 │\n│ num_applications ┆ 305137 ┆ 0 ┆ 0 │\n│ … ┆ … ┆ … ┆ … │\n│ amtdebitoutgoing_4809440A ┆ 1475550 ┆ 27286 ┆ 0 │\n│ amtdepositbalance_4809441A ┆ 1475550 ┆ 32235 ┆ 1 │\n│ amtdepositincoming_4809444A ┆ 1475550 ┆ 45985 ┆ 0 │\n│ amtdepositoutgoing_4809442A ┆ 1475550 ┆ 22433 ┆ 0 │\n│ has_other ┆ 0 ┆ 1475550 ┆ 0 │\n└─────────────────────────────┴────────────┴────────────┴────────────────┘","text/html":"
\nshape: (576, 4)
columnnull_countzero_countnegative_count
stri64i64i64
"case_id"010
"MONTH"000
"WEEK_NUM"0167350
"target"014786650
"num_applications"30513700
…………
"amtdebitoutgoing_4809440A"1475550272860
"amtdepositbalance_4809441A"1475550322351
"amtdepositincoming_4809444A"1475550459850
"amtdepositoutgoing_4809442A"1475550224330
"has_other"014755500
"},"metadata":{}}],"execution_count":20},{"cell_type":"code","source":"# Lọc ra danh sách cột kiểu chuỗi\nstring_cols = [col for col, dtype in train_merged.schema.items() if dtype == pl.Utf8]\n\n# In các giá trị unique cho từng cột chuỗi\nfor col in string_cols:\n uniques = train_merged.select(pl.col(col).unique().sort()).to_series()\n print(f\"\\n📌 Cột: {col} — {len(uniques)} giá trị unique:\")\n print(uniques)","metadata":{"_uuid":"2c01edfc-a4b2-44bc-9212-0c48c809d2f1","_cell_guid":"af9c1d35-0ec0-4cfb-b69d-789914b0f6d2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:51.421405Z","iopub.execute_input":"2025-06-23T20:39:51.421673Z","iopub.status.idle":"2025-06-23T20:39:52.601287Z","shell.execute_reply.started":"2025-06-23T20:39:51.42165Z","shell.execute_reply":"2025-06-23T20:39:52.597876Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"name":"stdout","text":"\n📌 Cột: cancelreason_3545846M_approved_mode — 31 giá trị unique:\nshape: (31,)\nSeries: 'cancelreason_3545846M_approved_mode' [str]\n[\n\tnull\n\t\"P107_145_100\"\n\t\"P118_30_169\"\n\t\"P11_56_131\"\n\t\"P120_0_10\"\n\t…\n\t\"P94_109_143\"\n\t\"P95_76_117\"\n\t\"P98_38_170\"\n\t\"P99_98_113\"\n\t\"a55475b1\"\n]\n\n📌 Cột: cancelreason_3545846M_rejected_mode — 68 giá trị unique:\nshape: (68,)\nSeries: 'cancelreason_3545846M_rejected_mode' [str]\n[\n\tnull\n\t\"P107_145_100\"\n\t\"P116_157_162\"\n\t\"P118_140_56\"\n\t\"P118_30_169\"\n\t…\n\t\"P94_154_184\"\n\t\"P98_38_170\"\n\t\"P99_98_113\"\n\t\"P9_82_76\"\n\t\"a55475b1\"\n]\n\n📌 Cột: cancelreason_3545846M_overall_mode — 67 giá trị unique:\nshape: (67,)\nSeries: 'cancelreason_3545846M_overall_mode' [str]\n[\n\tnull\n\t\"P107_145_100\"\n\t\"P116_157_162\"\n\t\"P118_140_56\"\n\t\"P118_30_169\"\n\t…\n\t\"P95_76_117\"\n\t\"P98_38_170\"\n\t\"P99_98_113\"\n\t\"P9_82_76\"\n\t\"a55475b1\"\n]\n\n📌 Cột: cancelreason_3545846M_last — 75 giá trị unique:\nshape: (75,)\nSeries: 'cancelreason_3545846M_last' [str]\n[\n\tnull\n\t\"P107_145_100\"\n\t\"P116_157_162\"\n\t\"P118_140_56\"\n\t\"P118_30_169\"\n\t…\n\t\"P95_76_117\"\n\t\"P98_38_170\"\n\t\"P99_98_113\"\n\t\"P9_82_76\"\n\t\"a55475b1\"\n]\n\n📌 Cột: credtype_587L_mode — 4 giá trị unique:\nshape: (4,)\nSeries: 'credtype_587L_mode' [str]\n[\n\tnull\n\t\"CAL\"\n\t\"COL\"\n\t\"REL\"\n]\n\n📌 Cột: credtype_587L_last — 4 giá trị unique:\nshape: (4,)\nSeries: 'credtype_587L_last' [str]\n[\n\tnull\n\t\"CAL\"\n\t\"COL\"\n\t\"REL\"\n]\n\n📌 Cột: education_1138M_mode — 7 giá trị unique:\nshape: (7,)\nSeries: 'education_1138M_mode' [str]\n[\n\tnull\n\t\"P106_81_188\"\n\t\"P157_18_172\"\n\t\"P17_36_170\"\n\t\"P33_146_175\"\n\t\"P97_36_170\"\n\t\"a55475b1\"\n]\n\n📌 Cột: education_1138M_last — 7 giá trị unique:\nshape: (7,)\nSeries: 'education_1138M_last' [str]\n[\n\tnull\n\t\"P106_81_188\"\n\t\"P157_18_172\"\n\t\"P17_36_170\"\n\t\"P33_146_175\"\n\t\"P97_36_170\"\n\t\"a55475b1\"\n]\n\n📌 Cột: familystate_726L_mode — 6 giá trị unique:\nshape: (6,)\nSeries: 'familystate_726L_mode' [str]\n[\n\tnull\n\t\"DIVORCED\"\n\t\"LIVING_WITH_PARTNER\"\n\t\"MARRIED\"\n\t\"SINGLE\"\n\t\"WIDOWED\"\n]\n\n📌 Cột: familystate_726L_last — 6 giá trị unique:\nshape: (6,)\nSeries: 'familystate_726L_last' [str]\n[\n\tnull\n\t\"DIVORCED\"\n\t\"LIVING_WITH_PARTNER\"\n\t\"MARRIED\"\n\t\"SINGLE\"\n\t\"WIDOWED\"\n]\n\n📌 Cột: inittransactioncode_279L_mode — 4 giá trị unique:\nshape: (4,)\nSeries: 'inittransactioncode_279L_mode' [str]\n[\n\tnull\n\t\"CASH\"\n\t\"NDF\"\n\t\"POS\"\n]\n\n📌 Cột: inittransactioncode_279L_last — 4 giá trị unique:\nshape: (4,)\nSeries: 'inittransactioncode_279L_last' [str]\n[\n\tnull\n\t\"CASH\"\n\t\"NDF\"\n\t\"POS\"\n]\n\n📌 Cột: postype_4733339M_mode — 10 giá trị unique:\nshape: (10,)\nSeries: 'postype_4733339M_mode' [str]\n[\n\tnull\n\t\"P140_48_169\"\n\t\"P149_40_170\"\n\t\"P169_115_83\"\n\t\"P177_117_192\"\n\t\"P217_110_186\"\n\t\"P46_145_78\"\n\t\"P60_146_156\"\n\t\"P67_102_161\"\n\t\"a55475b1\"\n]\n\n📌 Cột: postype_4733339M_last — 10 giá trị unique:\nshape: (10,)\nSeries: 'postype_4733339M_last' [str]\n[\n\tnull\n\t\"P140_48_169\"\n\t\"P149_40_170\"\n\t\"P169_115_83\"\n\t\"P177_117_192\"\n\t\"P217_110_186\"\n\t\"P46_145_78\"\n\t\"P60_146_156\"\n\t\"P67_102_161\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreason_755M_approved_mode — 17 giá trị unique:\nshape: (17,)\nSeries: 'rejectreason_755M_approved_mode' [str]\n[\n\tnull\n\t\"P121_60_164\"\n\t\"P129_162_80\"\n\t\"P198_131_9\"\n\t\"P19_25_34\"\n\t…\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"P99_56_166\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreason_755M_rejected_mode — 19 giá trị unique:\nshape: (19,)\nSeries: 'rejectreason_755M_rejected_mode' [str]\n[\n\tnull\n\t\"P121_60_164\"\n\t\"P129_162_80\"\n\t\"P185_59_178\"\n\t\"P196_88_176\"\n\t…\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"P99_56_166\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreason_755M_overall_mode — 19 giá trị unique:\nshape: (19,)\nSeries: 'rejectreason_755M_overall_mode' [str]\n[\n\tnull\n\t\"P121_60_164\"\n\t\"P129_162_80\"\n\t\"P185_59_178\"\n\t\"P196_88_176\"\n\t…\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"P99_56_166\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreason_755M_last — 19 giá trị unique:\nshape: (19,)\nSeries: 'rejectreason_755M_last' [str]\n[\n\tnull\n\t\"P121_60_164\"\n\t\"P129_162_80\"\n\t\"P185_59_178\"\n\t\"P196_88_176\"\n\t…\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"P99_56_166\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreasonclient_4145042M_approved_mode — 7 giá trị unique:\nshape: (7,)\nSeries: 'rejectreasonclient_4145042M_approved_mode' [str]\n[\n\tnull\n\t\"P129_162_80\"\n\t\"P30_86_84\"\n\t\"P52_67_90\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreasonclient_4145042M_rejected_mode — 13 giá trị unique:\nshape: (13,)\nSeries: 'rejectreasonclient_4145042M_rejected_mode' [str]\n[\n\tnull\n\t\"P129_162_80\"\n\t\"P19_25_34\"\n\t\"P30_86_84\"\n\t\"P52_67_90\"\n\t…\n\t\"P64_121_167\"\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreasonclient_4145042M_overall_mode — 12 giá trị unique:\nshape: (12,)\nSeries: 'rejectreasonclient_4145042M_overall_mode' [str]\n[\n\tnull\n\t\"P129_162_80\"\n\t\"P19_25_34\"\n\t\"P30_86_84\"\n\t\"P52_67_90\"\n\t…\n\t\"P64_121_167\"\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"a55475b1\"\n]\n\n📌 Cột: rejectreasonclient_4145042M_last — 15 giá trị unique:\nshape: (15,)\nSeries: 'rejectreasonclient_4145042M_last' [str]\n[\n\tnull\n\t\"P129_162_80\"\n\t\"P19_25_34\"\n\t\"P203_151_99\"\n\t\"P204_22_168\"\n\t…\n\t\"P64_121_167\"\n\t\"P69_72_116\"\n\t\"P84_14_61\"\n\t\"P94_109_143\"\n\t\"a55475b1\"\n]\n\n📌 Cột: status_219L_mode — 11 giá trị unique:\nshape: (11,)\nSeries: 'status_219L_mode' [str]\n[\n\tnull\n\t\"A\"\n\t\"D\"\n\t\"H\"\n\t\"K\"\n\t…\n\t\"N\"\n\t\"P\"\n\t\"Q\"\n\t\"S\"\n\t\"T\"\n]\n\n📌 Cột: status_219L_last — 12 giá trị unique:\nshape: (12,)\nSeries: 'status_219L_last' [str]\n[\n\tnull\n\t\"A\"\n\t\"D\"\n\t\"H\"\n\t\"K\"\n\t…\n\t\"P\"\n\t\"Q\"\n\t\"R\"\n\t\"S\"\n\t\"T\"\n]\n\n📌 Cột: cacccardblochreas_147M_mode — 9 giá trị unique:\nshape: (9,)\nSeries: 'cacccardblochreas_147M_mode' [str]\n[\n\tnull\n\t\"P133_119_56\"\n\t\"P17_56_144\"\n\t\"P19_60_110\"\n\t\"P201_63_60\"\n\t\"P23_105_103\"\n\t\"P33_145_161\"\n\t\"Unknown\"\n\t\"a55475b1\"\n]\n\n📌 Cột: credacc_cards_status_52L_mode — 8 giá trị unique:\nshape: (8,)\nSeries: 'credacc_cards_status_52L_mode' [str]\n[\n\tnull\n\t\"ACTIVE\"\n\t\"BLOCKED\"\n\t\"CANCELLED\"\n\t\"INACTIVE\"\n\t\"RENEWED\"\n\t\"UNCONFIRMED\"\n\t\"Unknown\"\n]\n\n📌 Cột: classificationofcontr_13M_mode — 12 giá trị unique:\nshape: (12,)\nSeries: 'classificationofcontr_13M_mode' [str]\n[\n\tnull\n\t\"00135d9c\"\n\t\"01f63ac8\"\n\t\"0d95a828\"\n\t\"1cf4e481\"\n\t…\n\t\"4a5a01e3\"\n\t\"87bdbcba\"\n\t\"a55475b1\"\n\t\"be7b251d\"\n\t\"ea6782cc\"\n]\n\n📌 Cột: contractst_545M_mode — 44 giá trị unique:\nshape: (44,)\nSeries: 'contractst_545M_mode' [str]\n[\n\tnull\n\t\"01f63ac8\"\n\t\"02699f0c\"\n\t\"0dc85f9d\"\n\t\"25c364c5\"\n\t…\n\t\"df63fbf8\"\n\t\"e2e7d341\"\n\t\"ec24545f\"\n\t\"f7394eb0\"\n\t\"fd624e63\"\n]\n\n📌 Cột: description_351M_mode — 13 giá trị unique:\nshape: (13,)\nSeries: 'description_351M_mode' [str]\n[\n\tnull\n\t\"0349102c\"\n\t\"0bfbf8f5\"\n\t\"0cb4d552\"\n\t\"153cfa61\"\n\t…\n\t\"6da7c7ed\"\n\t\"8a7423d5\"\n\t\"95decc86\"\n\t\"a55475b1\"\n\t\"f8e51f8d\"\n]\n\n📌 Cột: purposeofcred_426M_mode — 19 giá trị unique:\nshape: (19,)\nSeries: 'purposeofcred_426M_mode' [str]\n[\n\tnull\n\t\"164ee705\"\n\t\"2162d1a4\"\n\t\"28bfa260\"\n\t\"4039fc25\"\n\t…\n\t\"P188_162_121\"\n\t\"a55475b1\"\n\t\"b1285059\"\n\t\"e19fdece\"\n\t\"e8f3b178\"\n]\n\n📌 Cột: purposeofcred_874M_mode — 24 giá trị unique:\nshape: (24,)\nSeries: 'purposeofcred_874M_mode' [str]\n[\n\tnull\n\t\"164ee705\"\n\t\"2162d1a4\"\n\t\"27b6de28\"\n\t\"28bfa260\"\n\t…\n\t\"d11871e7\"\n\t\"d9ae1a0e\"\n\t\"e19fdece\"\n\t\"e8f3b178\"\n\t\"ee7d1eb8\"\n]\n\n📌 Cột: subjectrole_182M_mode — 9 giá trị unique:\nshape: (9,)\nSeries: 'subjectrole_182M_mode' [str]\n[\n\tnull\n\t\"15f04f45\"\n\t\"652d52e3\"\n\t\"71ddaa88\"\n\t\"P28_48_88\"\n\t\"a55475b1\"\n\t\"ab3c25cf\"\n\t\"be4fd70b\"\n\t\"daf49a8a\"\n]\n\n📌 Cột: subjectrole_93M_mode — 10 giá trị unique:\nshape: (10,)\nSeries: 'subjectrole_93M_mode' [str]\n[\n\tnull\n\t\"0c42a10e\"\n\t\"15f04f45\"\n\t\"652d52e3\"\n\t\"71ddaa88\"\n\t\"P28_48_88\"\n\t\"a55475b1\"\n\t\"ab3c25cf\"\n\t\"be4fd70b\"\n\t\"daf49a8a\"\n]\n\n📌 Cột: education_927M_mode — 6 giá trị unique:\nshape: (6,)\nSeries: 'education_927M_mode' [str]\n[\n\t\"P106_81_188\"\n\t\"P157_18_172\"\n\t\"P17_36_170\"\n\t\"P33_146_175\"\n\t\"P97_36_170\"\n\t\"a55475b1\"\n]\n\n📌 Cột: incometype_1044T_mode — 9 giá trị unique:\nshape: (9,)\nSeries: 'incometype_1044T_mode' [str]\n[\n\tnull\n\t\"EMPLOYED\"\n\t\"HANDICAPPED_2\"\n\t\"HANDICAPPED_3\"\n\t\"OTHER\"\n\t\"PRIVATE_SECTOR_EMPLOYEE\"\n\t\"RETIRED_PENSIONER\"\n\t\"SALARIED_GOVT\"\n\t\"SELFEMPLOYED\"\n]\n\n📌 Cột: language1_981M_mode — 3 giá trị unique:\nshape: (3,)\nSeries: 'language1_981M_mode' [str]\n[\n\t\"P10_39_147\"\n\t\"P209_127_106\"\n\t\"a55475b1\"\n]\n\n📌 Cột: role_1084L_mode — 4 giá trị unique:\nshape: (4,)\nSeries: 'role_1084L_mode' [str]\n[\n\tnull\n\t\"CL\"\n\t\"EM\"\n\t\"PE\"\n]\n\n📌 Cột: sex_738L_mode — 3 giá trị unique:\nshape: (3,)\nSeries: 'sex_738L_mode' [str]\n[\n\tnull\n\t\"F\"\n\t\"M\"\n]\n\n📌 Cột: type_25L_mode — 8 giá trị unique:\nshape: (8,)\nSeries: 'type_25L_mode' [str]\n[\n\tnull\n\t\"ALTERNATIVE_PHONE\"\n\t\"HOME_PHONE\"\n\t\"PHONE\"\n\t\"PRIMARY_EMAIL\"\n\t\"PRIMARY_MOBILE\"\n\t\"SECONDARY_MOBILE\"\n\t\"WHATSAPP\"\n]\n\n📌 Cột: conts_role_79M_mode — 11 giá trị unique:\nshape: (11,)\nSeries: 'conts_role_79M_mode' [str]\n[\n\tnull\n\t\"P115_147_77\"\n\t\"P124_137_181\"\n\t\"P125_105_50\"\n\t\"P125_14_176\"\n\t…\n\t\"P206_38_166\"\n\t\"P38_92_157\"\n\t\"P58_79_51\"\n\t\"P7_147_157\"\n\t\"a55475b1\"\n]\n\n📌 Cột: bankacctype_710L — 2 giá trị unique:\nshape: (2,)\nSeries: 'bankacctype_710L' [str]\n[\n\tnull\n\t\"CA\"\n]\n\n📌 Cột: cardtype_51L — 3 giá trị unique:\nshape: (3,)\nSeries: 'cardtype_51L' [str]\n[\n\tnull\n\t\"INSTANT\"\n\t\"PERSONALIZED\"\n]\n\n📌 Cột: credtype_322L — 4 giá trị unique:\nshape: (4,)\nSeries: 'credtype_322L' [str]\n[\n\tnull\n\t\"CAL\"\n\t\"COL\"\n\t\"REL\"\n]\n\n📌 Cột: disbursementtype_67L — 4 giá trị unique:\nshape: (4,)\nSeries: 'disbursementtype_67L' [str]\n[\n\tnull\n\t\"DD\"\n\t\"GBA\"\n\t\"SBA\"\n]\n\n📌 Cột: inittransactioncode_186L — 4 giá trị unique:\nshape: (4,)\nSeries: 'inittransactioncode_186L' [str]\n[\n\tnull\n\t\"CASH\"\n\t\"NDF\"\n\t\"POS\"\n]\n\n📌 Cột: paytype1st_925L — 2 giá trị unique:\nshape: (2,)\nSeries: 'paytype1st_925L' [str]\n[\n\tnull\n\t\"OTHER\"\n]\n\n📌 Cột: paytype_783L — 2 giá trị unique:\nshape: (2,)\nSeries: 'paytype_783L' [str]\n[\n\tnull\n\t\"OTHER\"\n]\n\n📌 Cột: twobodfilling_608L — 3 giá trị unique:\nshape: (3,)\nSeries: 'twobodfilling_608L' [str]\n[\n\tnull\n\t\"BO\"\n\t\"FO\"\n]\n\n📌 Cột: typesuite_864L — 2 giá trị unique:\nshape: (2,)\nSeries: 'typesuite_864L' [str]\n[\n\tnull\n\t\"AL\"\n]\n\n📌 Cột: education — 6 giá trị unique:\nshape: (6,)\nSeries: 'education' [str]\n[\n\tnull\n\t\"39a0853f\"\n\t\"6b2ae0fa\"\n\t\"717ddd49\"\n\t\"a55475b1\"\n\t\"c8e1a1d0\"\n]\n\n📌 Cột: marital_status — 7 giá trị unique:\nshape: (7,)\nSeries: 'marital_status' [str]\n[\n\tnull\n\t\"3439d993\"\n\t\"38c061ee\"\n\t\"a55475b1\"\n\t\"a7fcb6e5\"\n\t\"b6cabe76\"\n\t\"ecd83604\"\n]\n\n📌 Cột: description_5085714M — 3 giá trị unique:\nshape: (3,)\nSeries: 'description_5085714M' [str]\n[\n\tnull\n\t\"2fc785b2\"\n\t\"a55475b1\"\n]\n\n📌 Cột: requesttype_4525192L — 4 giá trị unique:\nshape: (4,)\nSeries: 'requesttype_4525192L' [str]\n[\n\tnull\n\t\"DEDUCTION_6\"\n\t\"PENSION_6\"\n\t\"SOCIAL_6\"\n]\n\n📌 Cột: riskassesment_302T — 17 giá trị unique:\nshape: (17,)\nSeries: 'riskassesment_302T' [str]\n[\n\tnull\n\t\"1% - 1%\"\n\t\"11% - 15%\"\n\t\"15% - 19%\"\n\t\"2% - 2%\"\n\t…\n\t\"50% - 58%\"\n\t\"59% - 66%\"\n\t\"6% - 8%\"\n\t\"67% - 100%\"\n\t\"8% - 11%\"\n]\n","output_type":"stream"}],"execution_count":21},{"cell_type":"code","source":"import polars as pl\n\n# Xác định các kiểu dữ liệu số\nnumeric_types = {\n pl.Int8, pl.Int16, pl.Int32, pl.Int64,\n pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,\n pl.Float32, pl.Float64\n}\n\n# Lấy schema\nschema = train_merged.schema\n\n# Phân loại cột\nnumeric_cols = [col for col, dtype in schema.items() if dtype in numeric_types]\nstring_cols = [col for col, dtype in schema.items() if dtype == pl.Utf8]\n\n# Fill null: số → -99999, chuỗi → \"Missing Value\" (đè lên cột gốc)\ntrain_merged = train_merged.with_columns(\n [pl.col(col).fill_null(-99999).alias(col) for col in numeric_cols] +\n [pl.col(col).fill_null(\"Missing Value\").alias(col) for col in string_cols]\n)","metadata":{"_uuid":"cc660159-70ae-424e-949c-2122784d78ee","_cell_guid":"c247d456-4f22-4400-a099-8c9bc7ad529c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:52.603902Z","iopub.execute_input":"2025-06-23T20:39:52.60438Z","iopub.status.idle":"2025-06-23T20:39:53.383767Z","shell.execute_reply.started":"2025-06-23T20:39:52.604355Z","shell.execute_reply":"2025-06-23T20:39:53.378123Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":22},{"cell_type":"code","source":"del applprev_features, credit_bureau_features, tax_features\ndel person_features, deposit_features, static_features\ndel static_cb_features, other_features","metadata":{"_uuid":"a5e182d3-74fb-407c-82e8-729ba9a1b2c9","_cell_guid":"eb73510e-27bb-481d-8415-18a81321547b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:53.38541Z","iopub.execute_input":"2025-06-23T20:39:53.385641Z","iopub.status.idle":"2025-06-23T20:39:54.641175Z","shell.execute_reply.started":"2025-06-23T20:39:53.385618Z","shell.execute_reply":"2025-06-23T20:39:54.634586Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":23},{"cell_type":"code","source":"summary = pl.DataFrame({\n \"column\": numeric_cols,\n \"null_count\": [train_merged[col].null_count() for col in numeric_cols],\n \"zero_count\": [(train_merged[col] == 0).sum() for col in numeric_cols],\n \"negative_count\": [(train_merged[col] < 0).sum() for col in numeric_cols],\n})\nsummary","metadata":{"_uuid":"fe0c4b35-c479-463e-95f4-d8b859b5d897","_cell_guid":"45e1c5a5-4abc-40ea-990a-8f83e6fdb53b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:54.64246Z","iopub.execute_input":"2025-06-23T20:39:54.64286Z","iopub.status.idle":"2025-06-23T20:39:56.485269Z","shell.execute_reply.started":"2025-06-23T20:39:54.642824Z","shell.execute_reply":"2025-06-23T20:39:56.48035Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"shape: (576, 4)\n┌─────────────────────────────┬────────────┬────────────┬────────────────┐\n│ column ┆ null_count ┆ zero_count ┆ negative_count │\n│ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ i64 ┆ i64 ┆ i64 │\n╞═════════════════════════════╪════════════╪════════════╪════════════════╡\n│ case_id ┆ 0 ┆ 1 ┆ 0 │\n│ MONTH ┆ 0 ┆ 0 ┆ 0 │\n│ WEEK_NUM ┆ 0 ┆ 16735 ┆ 0 │\n│ target ┆ 0 ┆ 1478665 ┆ 0 │\n│ num_applications ┆ 0 ┆ 0 ┆ 305137 │\n│ … ┆ … ┆ … ┆ … │\n│ amtdebitoutgoing_4809440A ┆ 0 ┆ 27286 ┆ 1475550 │\n│ amtdepositbalance_4809441A ┆ 0 ┆ 32235 ┆ 1475551 │\n│ amtdepositincoming_4809444A ┆ 0 ┆ 45985 ┆ 1475550 │\n│ amtdepositoutgoing_4809442A ┆ 0 ┆ 22433 ┆ 1475550 │\n│ has_other ┆ 0 ┆ 1475550 ┆ 0 │\n└─────────────────────────────┴────────────┴────────────┴────────────────┘","text/html":"
\nshape: (576, 4)
columnnull_countzero_countnegative_count
stri64i64i64
"case_id"010
"MONTH"000
"WEEK_NUM"0167350
"target"014786650
"num_applications"00305137
…………
"amtdebitoutgoing_4809440A"0272861475550
"amtdepositbalance_4809441A"0322351475551
"amtdepositincoming_4809444A"0459851475550
"amtdepositoutgoing_4809442A"0224331475550
"has_other"014755500
"},"metadata":{}}],"execution_count":24},{"cell_type":"code","source":"columns_to_fix = [\n \"overdue_ratio_active_mean\", \"overdue_ratio_active_max\", \"overdue_ratio_active_sum\",\n \"overdue_ratio_closed_mean\", \"overdue_ratio_closed_max\", \"overdue_ratio_closed_sum\",\n \"overdue_ratio_max_active_mean\", \"overdue_ratio_max_closed_mean\",\n \"dpd_ratio_closed_mean\", \"dpd_ratio_closed_max\",\n \"utilization_ratio_active_mean\", \"utilization_ratio_active_max\",\n \"utilization_ratio_closed_mean\", \"utilization_ratio_closed_max\",\n \"debt_to_limit_ratio_mean\", \"debt_to_limit_ratio_sum\",\n \"payment_regularity_active_mean\", \"payment_regularity_active_max\",\n \"payment_consistency_active_mean\", \"payment_consistency_active_max\",\n \"payment_consistency_closed_mean\", \"payment_consistency_closed_max\",\n \"payment_behavior_score_active_mean\", \"payment_behavior_score_active_max\",\n \"recent_overdue_trend_closed\", \"outstanding_trend_active_mean\", \"outstanding_trend_active_max\",\n \"outstanding_trend_closed_mean\", \"outstanding_trend_closed_max\",\n \"active_closed_ratio_mean\", \"active_closed_ratio_sum\",\n \"avg_contract_size_closed_mean\"\n]\n\n# Thay thế inf/-inf bằng -99999\nfor col in columns_to_fix:\n train_merged = train_merged.with_columns(\n pl.when(train_merged[col].is_infinite())\n .then(-99999)\n .otherwise(train_merged[col])\n .alias(col)\n )","metadata":{"_uuid":"e5461c00-d133-4af1-8745-efa26c2cb968","_cell_guid":"b065d6ac-61b4-4ced-a3a5-ca22290ce92e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:56.487398Z","iopub.execute_input":"2025-06-23T20:39:56.488928Z","iopub.status.idle":"2025-06-23T20:39:56.70135Z","shell.execute_reply.started":"2025-06-23T20:39:56.488898Z","shell.execute_reply":"2025-06-23T20:39:56.69687Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":25},{"cell_type":"code","source":"summary = pl.DataFrame({\n \"column\": numeric_cols,\n \"null_count\": [train_merged[col].null_count() for col in numeric_cols],\n \"zero_count\": [(train_merged[col] == 0).sum() for col in numeric_cols],\n \"negative_count\": [(train_merged[col] < 0).sum() for col in numeric_cols],\n \"inf_count\": [(train_merged[col].is_infinite()).sum() for col in numeric_cols]\n})\nsummary","metadata":{"_uuid":"ebdc42fe-de1b-400f-8b05-cf35697ee4a4","_cell_guid":"2259c506-bc3e-487f-80fa-0bd412b3dbcf","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:39:56.703233Z","iopub.execute_input":"2025-06-23T20:39:56.703461Z","iopub.status.idle":"2025-06-23T20:40:00.076802Z","shell.execute_reply.started":"2025-06-23T20:39:56.703439Z","shell.execute_reply":"2025-06-23T20:40:00.072851Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":26,"output_type":"execute_result","data":{"text/plain":"shape: (576, 5)\n┌─────────────────────────────┬────────────┬────────────┬────────────────┬───────────┐\n│ column ┆ null_count ┆ zero_count ┆ negative_count ┆ inf_count │\n│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n╞═════════════════════════════╪════════════╪════════════╪════════════════╪═══════════╡\n│ case_id ┆ 0 ┆ 1 ┆ 0 ┆ 0 │\n│ MONTH ┆ 0 ┆ 0 ┆ 0 ┆ 0 │\n│ WEEK_NUM ┆ 0 ┆ 16735 ┆ 0 ┆ 0 │\n│ target ┆ 0 ┆ 1478665 ┆ 0 ┆ 0 │\n│ num_applications ┆ 0 ┆ 0 ┆ 305137 ┆ 0 │\n│ … ┆ … ┆ … ┆ … ┆ … │\n│ amtdebitoutgoing_4809440A ┆ 0 ┆ 27286 ┆ 1475550 ┆ 0 │\n│ amtdepositbalance_4809441A ┆ 0 ┆ 32235 ┆ 1475551 ┆ 0 │\n│ amtdepositincoming_4809444A ┆ 0 ┆ 45985 ┆ 1475550 ┆ 0 │\n│ amtdepositoutgoing_4809442A ┆ 0 ┆ 22433 ┆ 1475550 ┆ 0 │\n│ has_other ┆ 0 ┆ 1475550 ┆ 0 ┆ 0 │\n└─────────────────────────────┴────────────┴────────────┴────────────────┴───────────┘","text/html":"
\nshape: (576, 5)
columnnull_countzero_countnegative_countinf_count
stri64i64i64i64
"case_id"0100
"MONTH"0000
"WEEK_NUM"01673500
"target"0147866500
"num_applications"003051370
……………
"amtdebitoutgoing_4809440A"02728614755500
"amtdepositbalance_4809441A"03223514755510
"amtdepositincoming_4809444A"04598514755500
"amtdepositoutgoing_4809442A"02243314755500
"has_other"0147555000
"},"metadata":{}}],"execution_count":26},{"cell_type":"code","source":"summary.filter(pl.col('null_count')>0)","metadata":{"_uuid":"f49e4d9a-15da-435c-a5d7-5742c7cc2712","_cell_guid":"5d05cc0a-c225-4bd2-9831-275d9e693370","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:40:00.079186Z","iopub.execute_input":"2025-06-23T20:40:00.079429Z","iopub.status.idle":"2025-06-23T20:40:00.094423Z","shell.execute_reply.started":"2025-06-23T20:40:00.079404Z","shell.execute_reply":"2025-06-23T20:40:00.087717Z"},"jupyter":{"outputs_hidden":false}},"outputs":[{"execution_count":27,"output_type":"execute_result","data":{"text/plain":"shape: (0, 5)\n┌────────┬────────────┬────────────┬────────────────┬───────────┐\n│ column ┆ null_count ┆ zero_count ┆ negative_count ┆ inf_count │\n│ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n│ str ┆ i64 ┆ i64 ┆ i64 ┆ i64 │\n╞════════╪════════════╪════════════╪════════════════╪═══════════╡\n└────────┴────────────┴────────────┴────────────────┴───────────┘","text/html":"
\nshape: (0, 5)
columnnull_countzero_countnegative_countinf_count
stri64i64i64i64
"},"metadata":{}}],"execution_count":27},{"cell_type":"code","source":"# train_merged.write_csv(\"train_merged.csv\")","metadata":{"_uuid":"f55b3802-c240-4526-9356-b230cff6f632","_cell_guid":"35f73b3e-3f23-4f43-8650-c4875c722929","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-23T20:40:00.096282Z","iopub.execute_input":"2025-06-23T20:40:00.096494Z","iopub.status.idle":"2025-06-23T20:40:00.110278Z","shell.execute_reply.started":"2025-06-23T20:40:00.096473Z","shell.execute_reply":"2025-06-23T20:40:00.10566Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":28},{"cell_type":"code","source":"train_merged","metadata":{"_uuid":"2cbe5cd8-0b8d-4e8a-aef8-aaf151531075","_cell_guid":"2f613154-a7ee-445d-91ca-27f5f1f7b321","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-06-23T20:40:00.112948Z","iopub.execute_input":"2025-06-23T20:40:00.113174Z","iopub.status.idle":"2025-06-23T20:40:00.137507Z","shell.execute_reply.started":"2025-06-23T20:40:00.113152Z","shell.execute_reply":"2025-06-23T20:40:00.1343Z"}},"outputs":[{"execution_count":29,"output_type":"execute_result","data":{"text/plain":"shape: (1_526_659, 633)\n┌─────────┬────────┬──────────┬────────┬───┬──────────────┬──────────────┬─────────────┬───────────┐\n│ case_id ┆ MONTH ┆ WEEK_NUM ┆ target ┆ … ┆ amtdepositba ┆ amtdepositin ┆ amtdeposito ┆ has_other │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ lance_480944 ┆ coming_48094 ┆ utgoing_480 ┆ --- │\n│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ 1A ┆ 44A ┆ 9442A ┆ i32 │\n│ ┆ ┆ ┆ ┆ ┆ --- ┆ --- ┆ --- ┆ │\n│ ┆ ┆ ┆ ┆ ┆ f64 ┆ f64 ┆ f64 ┆ │\n╞═════════╪════════╪══════════╪════════╪═══╪══════════════╪══════════════╪═════════════╪═══════════╡\n│ 0 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 1 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 2 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 3 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 4 ┆ 201901 ┆ 0 ┆ 1 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n│ 2703450 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 1 │\n│ 2703451 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 1 │\n│ 2703452 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 2703453 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 822.0 ┆ 0.0 ┆ 6.8 ┆ 1 │\n│ 2703454 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n└─────────┴────────┴──────────┴────────┴───┴──────────────┴──────────────┴─────────────┴───────────┘","text/html":"
\nshape: (1_526_659, 633)
case_idMONTHWEEK_NUMtargetnum_applicationsnum_approvedrate_approval_applicationactualdpd_943P_approved_maxactualdpd_943P_approved_avgactualdpd_943P_approved_minactualdpd_943P_approved_lastannuity_853A_approved_maxannuity_853A_approved_avgannuity_853A_approved_minannuity_853A_approved_lastannuity_853A_rejected_maxannuity_853A_rejected_avgannuity_853A_rejected_minannuity_853A_rejected_stdannuity_853A_rejected_lastannuity_853A_lastapprovaldate_319D_durationapprovaldate_319D_intervalbyoccupationinc_3656910L_maxbyoccupationinc_3656910L_avgbyoccupationinc_3656910L_minbyoccupationinc_3656910L_stdbyoccupationinc_3656910L_lastcancelreason_3545846M_approved_modecancelreason_3545846M_rejected_modecancelreason_3545846M_overall_modecancelreason_3545846M_lastchildnum_21L_approved_avgchildnum_21L_rejected_avgchildnum_21L_lastcreationdate_885D_duration_allcreationdate_885D_interval_approved…formonth_118Lformonth_206Lformonth_535Lforquarter_1017Lforquarter_462Lforquarter_634Lfortoday_1092Lforweek_1077Lforweek_528Lforweek_601Lforyear_618Lforyear_818Lforyear_850Lfourthquarter_440Lnumberofqueries_373Lpmtscount_423Lpmtssum_45Arequesttype_4525192Lriskassesment_302Triskassesment_940Tsecondquarter_766Lthirdquarter_1082Lage_estimateprocessing_dayscredit_activity_ratiomidterm_credit_ratiorejection_densityshortterm_rejection_densitytax_complianceresponse_date_date_decisionhas_static_cbamtdebitincoming_4809443Aamtdebitoutgoing_4809440Aamtdepositbalance_4809441Aamtdepositincoming_4809444Aamtdepositoutgoing_4809442Ahas_other
i64i64i64i64i64i64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64i64f64f64f64f64f64f64strstrstrstrf64f64f64i64f64…f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64strstrf64f64f64i64i64f64f64f64f64f64i64i32f64f64f64f64f64i32
020190100-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value""Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
120190100-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value""Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
220190100200.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.01682.41161.3640.2736.9466871682.41682.4-999990.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""a55475b1""a55475b1""a55475b1"-99999.00.00.000.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
320190100100.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.06140.06140.06140.0-99999.06140.06140.0-999990.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""P94_109_143""P94_109_143""P94_109_143"-99999.0-99999.0-99999.000.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
420190101100.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.02556.62556.62556.6-99999.02556.62556.6-999990.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""P24_27_36""P24_27_36""P24_27_36"-99999.0-99999.0-99999.000.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
………………………………………………………………………………………………………………………………………………………………………………………………………
270345020201091013110.8461540.00.00.00.04965.22528.0000180.02102.230875.030875.030875.0-99999.030875.02102.24502450.21.00.50.00.707107-99999.0"a55475b1""P94_109_143""a55475b1""a55475b1"0.333333-99999.0-99999.04502450.2…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.01.00.0-99999.0-99999.0"Missing Value""Missing Value"-99999.01.01.05810120.00.0-99999.0-99999.01.01410.00.00.00.00.01
2703451202010910661.00.00.00.00.012809.26472.8666670.06191.6-99999.0-99999.0-99999.0-99999.0-99999.06191.61536307.210340.010340.010340.0-99999.0-99999.0"a55475b1""Missing Value""a55475b1""a55475b1"0.0-99999.0-99999.01536307.2…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.00.00.0-99999.0-99999.0"Missing Value""Missing Value"-99999.01.02.05556050.00.0-99999.0-99999.01.014127500.027477.60.00.00.01
2703452202010910320.6666670.00.00.00.03243.40012372.200051501.01501.09048.09048.09048.0-99999.09048.09048.0337337.0-99999.0-99999.0-99999.0-99999.0-99999.0"a55475b1""P180_60_137""a55475b1""P180_60_137"-99999.0-99999.0-99999.0705337.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.01.03.0-99999.0-99999.0"Missing Value""Missing Value"-99999.00.04.0-99999-999990.00.666667-99999.0-99999.0-99999.0141-99999.0-99999.0-99999.0-99999.0-99999.00
2703453202010910980.8888890.00.00.00.05981.41962.1250020.02827.2416.2416.2416.2-99999.0416.22827.22363337.57142933059.033059.033059.0-99999.0-99999.0"a55475b1""a55475b1""a55475b1""a55475b1"0.00.0-99999.02363337.571429…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.03.04.0-99999.0-99999.0"Missing Value""Missing Value"-99999.02.01.05846280.250.5-99999.0-99999.01.16666712113454.013333.4822.00.06.81
2703454202010910221.00.00.00.00.06726.64856.72986.82986.8-99999.0-99999.0-99999.0-99999.0-99999.02986.8325325.0-99999.0-99999.0-99999.0-99999.0-99999.0"a55475b1""Missing Value""a55475b1""a55475b1"-99999.0-99999.0-99999.0325325.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.00.01.0-99999.0-99999.0"Missing Value""Missing Value"-99999.02.01.05273630.01.0-99999.0-99999.01.0141-99999.0-99999.0-99999.0-99999.0-99999.00
"},"metadata":{}}],"execution_count":29},{"cell_type":"code","source":"","metadata":{"_uuid":"922c0224-ba3f-4216-bfb4-a652e3f59c15","_cell_guid":"e2bbd5fa-abda-43d6-b9f6-b29cfa907141","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_merged","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:40:00.141016Z","iopub.execute_input":"2025-06-23T20:40:00.141638Z","iopub.status.idle":"2025-06-23T20:40:00.169765Z","shell.execute_reply.started":"2025-06-23T20:40:00.141613Z","shell.execute_reply":"2025-06-23T20:40:00.163425Z"}},"outputs":[{"execution_count":30,"output_type":"execute_result","data":{"text/plain":"shape: (1_526_659, 633)\n┌─────────┬────────┬──────────┬────────┬───┬──────────────┬──────────────┬─────────────┬───────────┐\n│ case_id ┆ MONTH ┆ WEEK_NUM ┆ target ┆ … ┆ amtdepositba ┆ amtdepositin ┆ amtdeposito ┆ has_other │\n│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ lance_480944 ┆ coming_48094 ┆ utgoing_480 ┆ --- │\n│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ ┆ 1A ┆ 44A ┆ 9442A ┆ i32 │\n│ ┆ ┆ ┆ ┆ ┆ --- ┆ --- ┆ --- ┆ │\n│ ┆ ┆ ┆ ┆ ┆ f64 ┆ f64 ┆ f64 ┆ │\n╞═════════╪════════╪══════════╪════════╪═══╪══════════════╪══════════════╪═════════════╪═══════════╡\n│ 0 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 1 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 2 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 3 ┆ 201901 ┆ 0 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 4 ┆ 201901 ┆ 0 ┆ 1 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n│ 2703450 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 1 │\n│ 2703451 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 1 │\n│ 2703452 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n│ 2703453 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ 822.0 ┆ 0.0 ┆ 6.8 ┆ 1 │\n│ 2703454 ┆ 202010 ┆ 91 ┆ 0 ┆ … ┆ -99999.0 ┆ -99999.0 ┆ -99999.0 ┆ 0 │\n└─────────┴────────┴──────────┴────────┴───┴──────────────┴──────────────┴─────────────┴───────────┘","text/html":"
\nshape: (1_526_659, 633)
case_idMONTHWEEK_NUMtargetnum_applicationsnum_approvedrate_approval_applicationactualdpd_943P_approved_maxactualdpd_943P_approved_avgactualdpd_943P_approved_minactualdpd_943P_approved_lastannuity_853A_approved_maxannuity_853A_approved_avgannuity_853A_approved_minannuity_853A_approved_lastannuity_853A_rejected_maxannuity_853A_rejected_avgannuity_853A_rejected_minannuity_853A_rejected_stdannuity_853A_rejected_lastannuity_853A_lastapprovaldate_319D_durationapprovaldate_319D_intervalbyoccupationinc_3656910L_maxbyoccupationinc_3656910L_avgbyoccupationinc_3656910L_minbyoccupationinc_3656910L_stdbyoccupationinc_3656910L_lastcancelreason_3545846M_approved_modecancelreason_3545846M_rejected_modecancelreason_3545846M_overall_modecancelreason_3545846M_lastchildnum_21L_approved_avgchildnum_21L_rejected_avgchildnum_21L_lastcreationdate_885D_duration_allcreationdate_885D_interval_approved…formonth_118Lformonth_206Lformonth_535Lforquarter_1017Lforquarter_462Lforquarter_634Lfortoday_1092Lforweek_1077Lforweek_528Lforweek_601Lforyear_618Lforyear_818Lforyear_850Lfourthquarter_440Lnumberofqueries_373Lpmtscount_423Lpmtssum_45Arequesttype_4525192Lriskassesment_302Triskassesment_940Tsecondquarter_766Lthirdquarter_1082Lage_estimateprocessing_dayscredit_activity_ratiomidterm_credit_ratiorejection_densityshortterm_rejection_densitytax_complianceresponse_date_date_decisionhas_static_cbamtdebitincoming_4809443Aamtdebitoutgoing_4809440Aamtdepositbalance_4809441Aamtdepositincoming_4809444Aamtdepositoutgoing_4809442Ahas_other
i64i64i64i64i64i64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64i64f64f64f64f64f64f64strstrstrstrf64f64f64i64f64…f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64f64strstrf64f64f64i64i64f64f64f64f64f64i64i32f64f64f64f64f64i32
020190100-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value""Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
120190100-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value""Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
220190100200.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.01682.41161.3640.2736.9466871682.41682.4-999990.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""a55475b1""a55475b1""a55475b1"-99999.00.00.000.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
320190100100.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.06140.06140.06140.0-99999.06140.06140.0-999990.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""P94_109_143""P94_109_143""P94_109_143"-99999.0-99999.0-99999.000.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
420190101100.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.02556.62556.62556.6-99999.02556.62556.6-999990.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""P24_27_36""P24_27_36""P24_27_36"-99999.0-99999.0-99999.000.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0"Missing Value""Missing Value"-99999.0-99999.0-99999.0-99999-99999-99999.0-99999.0-99999.0-99999.0-99999.0-999990-99999.0-99999.0-99999.0-99999.0-99999.00
………………………………………………………………………………………………………………………………………………………………………………………………………
270345020201091013110.8461540.00.00.00.04965.22528.0000180.02102.230875.030875.030875.0-99999.030875.02102.24502450.21.00.50.00.707107-99999.0"a55475b1""P94_109_143""a55475b1""a55475b1"0.333333-99999.0-99999.04502450.2…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.01.00.0-99999.0-99999.0"Missing Value""Missing Value"-99999.01.01.05810120.00.0-99999.0-99999.01.01410.00.00.00.00.01
2703451202010910661.00.00.00.00.012809.26472.8666670.06191.6-99999.0-99999.0-99999.0-99999.0-99999.06191.61536307.210340.010340.010340.0-99999.0-99999.0"a55475b1""Missing Value""a55475b1""a55475b1"0.0-99999.0-99999.01536307.2…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.00.00.0-99999.0-99999.0"Missing Value""Missing Value"-99999.01.02.05556050.00.0-99999.0-99999.01.014127500.027477.60.00.00.01
2703452202010910320.6666670.00.00.00.03243.40012372.200051501.01501.09048.09048.09048.0-99999.09048.09048.0337337.0-99999.0-99999.0-99999.0-99999.0-99999.0"a55475b1""P180_60_137""a55475b1""P180_60_137"-99999.0-99999.0-99999.0705337.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.01.03.0-99999.0-99999.0"Missing Value""Missing Value"-99999.00.04.0-99999-999990.00.666667-99999.0-99999.0-99999.0141-99999.0-99999.0-99999.0-99999.0-99999.00
2703453202010910980.8888890.00.00.00.05981.41962.1250020.02827.2416.2416.2416.2-99999.0416.22827.22363337.57142933059.033059.033059.0-99999.0-99999.0"a55475b1""a55475b1""a55475b1""a55475b1"0.00.0-99999.02363337.571429…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.03.04.0-99999.0-99999.0"Missing Value""Missing Value"-99999.02.01.05846280.250.5-99999.0-99999.01.16666712113454.013333.4822.00.06.81
2703454202010910221.00.00.00.00.06726.64856.72986.82986.8-99999.0-99999.0-99999.0-99999.0-99999.02986.8325325.0-99999.0-99999.0-99999.0-99999.0-99999.0"a55475b1""Missing Value""a55475b1""a55475b1"-99999.0-99999.0-99999.0325325.0…-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.0-99999.00.01.0-99999.0-99999.0"Missing Value""Missing Value"-99999.02.01.05273630.01.0-99999.0-99999.01.0141-99999.0-99999.0-99999.0-99999.0-99999.00
"},"metadata":{}}],"execution_count":30},{"cell_type":"code","source":"import polars as pl\n\n# Chuyển từ polars sang pandas\ndf = train_merged.to_pandas()\n\n","metadata":{"_uuid":"d534073f-acc2-4f4f-bb9d-b0f0d72d52e5","_cell_guid":"a40a4aab-f0a7-4c29-adc1-9a9081181bfa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-06-23T20:42:17.156476Z","iopub.execute_input":"2025-06-23T20:42:17.156842Z","iopub.status.idle":"2025-06-23T20:42:19.782991Z","shell.execute_reply.started":"2025-06-23T20:42:17.156809Z","shell.execute_reply":"2025-06-23T20:42:19.777583Z"}},"outputs":[],"execution_count":39},{"cell_type":"code","source":"# Đếm số lượng cột số\nnum_cols = df.select_dtypes(include=[\"number\"]).columns\nnum_count = len(num_cols)\n\n# Đếm số lượng cột chữ\ncat_cols = df.select_dtypes(include=[\"object\", \"string\", \"category\"]).columns\ncat_count = len(cat_cols)\n\nprint(f\"Số cột số: {num_count}\")\nprint(f\"Số cột chữ: {cat_count}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:42:19.785016Z","iopub.execute_input":"2025-06-23T20:42:19.785237Z","iopub.status.idle":"2025-06-23T20:42:21.796055Z","shell.execute_reply.started":"2025-06-23T20:42:19.785216Z","shell.execute_reply":"2025-06-23T20:42:21.79136Z"}},"outputs":[{"name":"stdout","text":"Số cột số: 576\nSố cột chữ: 57\n","output_type":"stream"}],"execution_count":40},{"cell_type":"code","source":"# 1. Xoá các cột không cần\ncols_to_drop = [\n \"isbidproduct_390L_approved_last\",\n \"isbidproduct_390L_rejected_last\",\n \"recent_update_closed\",\n \"recent_overdue_trend_active\",\n \"recent_overdue_trend_closed\"\n]\ndf = df.drop(columns=cols_to_drop, errors=\"ignore\")\n# 2. Lấy danh sách cột chữ và cột số\nnum_cols = df.select_dtypes(include=[\"number\"]).columns\ncat_cols = df.select_dtypes(include=[\"object\", \"string\", \"category\"]).columns\n\n# 3. Điền missing value\ndf[num_cols] = df[num_cols].fillna(-99999)\ndf[cat_cols] = df[cat_cols].fillna(\"Missing Value\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:42:23.6931Z","iopub.execute_input":"2025-06-23T20:42:23.693417Z","iopub.status.idle":"2025-06-23T20:42:46.177566Z","shell.execute_reply.started":"2025-06-23T20:42:23.693393Z","shell.execute_reply":"2025-06-23T20:42:46.173085Z"}},"outputs":[],"execution_count":41},{"cell_type":"code","source":"null_counts = df.isnull().sum()\nnull_counts = null_counts[null_counts > 0]\nprint(null_counts)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:42:46.179836Z","iopub.execute_input":"2025-06-23T20:42:46.180073Z","iopub.status.idle":"2025-06-23T20:42:50.757028Z","shell.execute_reply.started":"2025-06-23T20:42:46.180052Z","shell.execute_reply":"2025-06-23T20:42:50.751358Z"}},"outputs":[{"name":"stdout","text":"Series([], dtype: int64)\n","output_type":"stream"}],"execution_count":42},{"cell_type":"code","source":"!pip install fastsparsegams xgboost\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:42:50.758591Z","iopub.execute_input":"2025-06-23T20:42:50.75885Z","iopub.status.idle":"2025-06-23T20:42:55.24723Z","shell.execute_reply.started":"2025-06-23T20:42:50.758828Z","shell.execute_reply":"2025-06-23T20:42:55.242286Z"}},"outputs":[{"name":"stdout","text":"Requirement already satisfied: fastsparsegams in /usr/local/lib/python3.10/site-packages (0.2.0)\nRequirement already satisfied: xgboost in /usr/local/lib/python3.10/site-packages (3.0.2)\nRequirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.10/site-packages (from fastsparsegams) (1.15.2)\nRequirement already satisfied: pandas>=1.0.0 in /usr/local/lib/python3.10/site-packages (from fastsparsegams) (2.2.3)\nRequirement already satisfied: matplotlib in /usr/local/lib/python3.10/site-packages (from fastsparsegams) (3.10.1)\nRequirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/site-packages (from fastsparsegams) (2.0.2)\nRequirement already satisfied: nvidia-nccl-cu12 in /usr/local/lib/python3.10/site-packages (from xgboost) (2.21.5)\nRequirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/site-packages (from pandas>=1.0.0->fastsparsegams) (2.9.0.post0)\nRequirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/site-packages (from pandas>=1.0.0->fastsparsegams) (2025.2)\nRequirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas>=1.0.0->fastsparsegams) (2025.2)\nRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (25.0)\nRequirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (1.4.8)\nRequirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (3.2.3)\nRequirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (1.3.2)\nRequirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (0.12.1)\nRequirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (4.57.0)\nRequirement already satisfied: pillow>=8 in /usr/local/lib/python3.10/site-packages (from matplotlib->fastsparsegams) (11.2.1)\nRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=1.0.0->fastsparsegams) (1.17.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.1.1\u001b[0m\n\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n","output_type":"stream"}],"execution_count":43},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import accuracy_score\nimport fastsparsegams\n\n# Giả sử bạn đã cung cấp train_merged với cột 'case_id' và 'target'\n# Tách features và labels\ndf = df.drop(columns=[\"case_id\"]) # bỏ id không dùng để học\n\n# Encode target nếu chưa là số\ny = df[\"target\"]\nif y.dtype == \"object\":\n y = LabelEncoder().fit_transform(y)\nX = df.drop(columns=[\"target\"])\n\n# Đảm bảo X là số\nX = X.select_dtypes(include=[np.number]).fillna(0)\ny = np.array(y)\n# 1. XGBoost feature importance\nxgb_model = XGBClassifier(use_label_encoder=False, eval_metric=\"logloss\")\nxgb_model.fit(X, y)\n\nxgb_importance = pd.DataFrame({\n \"feature\": X.columns,\n \"importance\": xgb_model.feature_importances_\n}).sort_values(\"importance\", ascending=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:42:55.249849Z","iopub.execute_input":"2025-06-23T20:42:55.25012Z","iopub.status.idle":"2025-06-23T20:43:40.252198Z","shell.execute_reply.started":"2025-06-23T20:42:55.250083Z","shell.execute_reply":"2025-06-23T20:43:40.24627Z"}},"outputs":[{"name":"stderr","text":"/usr/local/lib/python3.10/site-packages/xgboost/training.py:183: UserWarning: [20:43:19] WARNING: /workspace/src/learner.cc:738: \nParameters: { \"use_label_encoder\" } are not used.\n\n bst.update(dtrain, iteration=i, fobj=obj)\n","output_type":"stream"}],"execution_count":44},{"cell_type":"code","source":"xgb_importance","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:43:47.155238Z","iopub.execute_input":"2025-06-23T20:43:47.155606Z","iopub.status.idle":"2025-06-23T20:43:47.187791Z","shell.execute_reply.started":"2025-06-23T20:43:47.155577Z","shell.execute_reply":"2025-06-23T20:43:47.182555Z"}},"outputs":[{"execution_count":45,"output_type":"execute_result","data":{"text/plain":" feature importance\n304 dpd_normalized_active_mean 0.132250\n404 avgdpdtolclosure24_3658938P 0.063905\n157 dpdmax_757P_mean 0.020610\n306 dpd_normalized_closed_mean 0.019951\n452 maxdpdlast12m_727P 0.018301\n.. ... ...\n537 formonth_118L 0.000000\n524 mastercontrexist_109L 0.000000\n525 has_static 0.000000\n68 downpmt_134A_rejected_min 0.000000\n571 has_other 0.000000\n\n[572 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
featureimportance
304dpd_normalized_active_mean0.132250
404avgdpdtolclosure24_3658938P0.063905
157dpdmax_757P_mean0.020610
306dpd_normalized_closed_mean0.019951
452maxdpdlast12m_727P0.018301
.........
537formonth_118L0.000000
524mastercontrexist_109L0.000000
525has_static0.000000
68downpmt_134A_rejected_min0.000000
571has_other0.000000
\n

572 rows × 2 columns

\n
"},"metadata":{}}],"execution_count":45},{"cell_type":"code","source":"# 2. FastSparseGAM importance (dựa trên sum(abs(coeff)))\nfit_model = fastsparsegams.fit(\n X.values.astype(float), y.astype(float),\n loss=\"Logistic\",\n penalty=\"L0\",\n max_support_size=100000,\n algorithm=\"CDPSI\"\n)\nlambda0 = fit_model.lambda_0[0][-1]\ncoeffs = fit_model.coeff(lambda_0=lambda0).toarray().ravel()\nlows = fit_model.lows\n\nimportance_list = []\nidx = 0\nfor j, fname in enumerate(X.columns):\n cnt = len(lows[j])\n coeff_slice = coeffs[idx: idx + cnt]\n importance = np.sum(np.abs(coeff_slice))\n importance_list.append((fname, importance))\n idx += cnt\n\nfastgam_importance = pd.DataFrame(importance_list, columns=[\"feature\", \"importance\"])\nfastgam_importance = fastgam_importance.sort_values(\"importance\", ascending=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:43:59.614068Z","iopub.execute_input":"2025-06-23T20:43:59.61442Z","execution_failed":"2025-06-23T22:40:54.235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fastgam_importance","metadata":{"trusted":true,"execution":{"execution_failed":"2025-06-23T22:40:54.237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.model_selection import train_test_split\n# Bước 1: Xác định các cột cần one-hot (cột kiểu object/string)\n# Lấy các cột dạng object hoặc string để one-hot\ncategorical_cols = df.select_dtypes(include=[\"object\", \"string\", \"category\"]).columns.tolist()\n\n# Định nghĩa bộ biến đổi\npreprocessor = ColumnTransformer(\n transformers=[\n (\"cat\", OneHotEncoder(handle_unknown=\"ignore\", sparse_output=False), categorical_cols)\n ],\n remainder=\"passthrough\"\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-23T20:40:59.898113Z","iopub.execute_input":"2025-06-23T20:40:59.898362Z","iopub.status.idle":"2025-06-23T20:41:03.352096Z","shell.execute_reply.started":"2025-06-23T20:40:59.898334Z","shell.execute_reply":"2025-06-23T20:41:03.34739Z"}},"outputs":[],"execution_count":37},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import train_test_split\nfrom gosdt import ThresholdGuessBinarizer, GOSDTClassifier\n\n# # Parameters\n# GBDT_N_EST = 40\n# GBDT_MAX_DEPTH = 1\n# REGULARIZATION = 0.001\n# SIMILAR_SUPPORT = False\n# DEPTH_BUDGET = 6\n# TIME_LIMIT = 60\n# VERBOSE = True\n\n# Read the dataset\nX = df.drop(columns=[\"case_id\", \"target\"])\ny = df[\"target\"]\n\n\n# Train test split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2021)\nprint(\"X train shape:{}, X test shape:{}\".format(X_train.shape, X_test.shape))\n\n# Fit trên train, transform cả train/test\nX_train = preprocessor.fit_transform(X_train)\nX_test = preprocessor.transform(X_test)\n\n# Lấy tên các cột sau khi one-hot (nếu cần)\nencoded_feature_names = preprocessor.get_feature_names_out()\n\n# Step 1: Guess Thresholds\n# X_train = pd.DataFrame(X_train, columns=h)\n# X_test = pd.DataFrame(X_test, columns=h)\nX_train = pd.DataFrame(X_train, columns=encoded_feature_names)\nX_test = pd.DataFrame(X_test, columns=encoded_feature_names)\nenc = ThresholdGuessBinarizer(n_estimators=40, max_depth=2, random_state=2021)\nenc.set_output(transform=\"pandas\")\nX_train_guessed = enc.fit_transform(X_train, y_train)\nX_test_guessed = enc.transform(X_test)\nprint(f\"After guessing, X train shape:{X_train_guessed.shape}, X test shape:{X_test_guessed.shape}\")\nprint(\"train set column names == test set column names: {list(X_train_guessed.columns)==list(X_test_guessed.columns)}\")\n\n","metadata":{"_uuid":"209788b7-9c49-4ec4-8746-a0410d29831a","_cell_guid":"3af988c2-046d-46c9-9caf-51f20f754908","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-06-18T05:24:30.72402Z","iopub.execute_input":"2025-06-18T05:24:30.724265Z"}},"outputs":[{"name":"stdout","text":"X train shape:(1221327, 626), X test shape:(305332, 626)\n","output_type":"stream"}],"execution_count":null},{"cell_type":"code","source":"# Step 2: Guess Lower Bounds\nenc = GradientBoostingClassifier(n_estimators=40, max_depth=22, random_state=42)\nenc.fit(X_train_guessed, y_train)\nwarm_labels = enc.predict(X_train_guessed)\n","metadata":{"_uuid":"1ff530de-86ca-4d1c-a60d-83fb3688091e","_cell_guid":"b919c616-74d1-4167-aa6e-be5f303cdb61","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3: Train the GOSDT classifier\nclf = GOSDTClassifier(regularization=REGULARIZATION, similar_support=SIMILAR_SUPPORT, time_limit=TIME_LIMIT, depth_budget=DEPTH_BUDGET, verbose=VERBOSE) \nclf.fit(X_train_guessed, y_train, y_ref=warm_labels)\n\n# Step 4: Evaluate the model\nprint(\"Evaluating the model, extracting tree and scores\", flush=True)\n\n\nprint(f\"Model training time: {clf.result_.time}\")\nprint(f\"Training accuracy: {clf.score(X_train_guessed, y_train)}\")\nprint(f\"Test accuracy: {clf.score(X_test_guessed, y_test)}\")","metadata":{"_uuid":"613f9a77-084b-4edf-a3ee-2ea8e53fbc77","_cell_guid":"e2562d77-1ed8-4e1f-8a64-2d58357879fc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"094cf8ac-3ac5-45d8-b81f-f1c2c15e33f7","_cell_guid":"0cc63d21-8866-45d1-8793-18aa98b9bd6e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"ae8971ec-b7cf-4bdc-8bd7-af9b81acac3b","_cell_guid":"57fab2db-7cf4-4bea-8019-4526695c00aa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"d2b36cd3-40f4-4231-9a6c-b7a8721862e8","_cell_guid":"a2ec683e-1563-4b1a-a229-a18f15a72144","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"_uuid":"85879d86-eb95-4775-b269-5309b2ea84c9","_cell_guid":"9896816a-b6f7-48e6-94e8-d5a84c0486e1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import shutil\n\nshutil.copytree(\n \"/kaggle/input/gosdt-final/GeneralizedOptimalSparseDecisionTrees-master\",\n \"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master\"\n)","metadata":{"_uuid":"4a0e2cf2-9704-474c-8787-5ef370d766f2","_cell_guid":"79445c3f-1162-4886-98f2-f91f26b86aa0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:41:21.07281Z","iopub.execute_input":"2025-06-16T09:41:21.07316Z","iopub.status.idle":"2025-06-16T09:41:39.104277Z","shell.execute_reply.started":"2025-06-16T09:41:21.073129Z","shell.execute_reply":"2025-06-16T09:41:39.098554Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!apt-get update\n!apt-get install -y libtbb-dev libgmp-dev ocl-icd-opencl-dev","metadata":{"_uuid":"a32cb1ef-1bed-4dc4-ab59-6e7bc9533d62","_cell_guid":"55011b14-961b-4573-94a4-0ce7ae6ed0d1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:41:50.993438Z","iopub.execute_input":"2025-06-16T09:41:50.993778Z","iopub.status.idle":"2025-06-16T09:41:58.068855Z","shell.execute_reply.started":"2025-06-16T09:41:50.99374Z","shell.execute_reply":"2025-06-16T09:41:58.064483Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python3 /kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/setup.py build\n!python3 /kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/setup.py install --user","metadata":{"_uuid":"a6249a1d-61e6-4bee-a10f-0ca1de0fb10b","_cell_guid":"cb31144e-ac40-460b-b1b2-9b3891ca69c7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:42:59.054182Z","iopub.execute_input":"2025-06-16T09:42:59.05447Z","iopub.status.idle":"2025-06-16T09:43:00.069779Z","shell.execute_reply.started":"2025-06-16T09:42:59.05444Z","shell.execute_reply":"2025-06-16T09:43:00.063215Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cd \"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master\"","metadata":{"_uuid":"7bdc3aa6-9e23-4682-b92c-4523de72d998","_cell_guid":"3bef8103-c875-4b23-afe9-e0d8c00177b8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:45:11.303615Z","iopub.execute_input":"2025-06-16T09:45:11.303916Z","iopub.status.idle":"2025-06-16T09:45:11.317413Z","shell.execute_reply.started":"2025-06-16T09:45:11.303885Z","shell.execute_reply":"2025-06-16T09:45:11.310823Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install sortedcontainers gmpy2","metadata":{"_uuid":"f4358cac-28da-48cb-bd9f-eb332cffb097","_cell_guid":"931fdf0c-aa8f-4ec8-a9df-ae7af1a80ba8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:43:15.00652Z","iopub.execute_input":"2025-06-16T09:43:15.006818Z","iopub.status.idle":"2025-06-16T09:43:20.537185Z","shell.execute_reply.started":"2025-06-16T09:43:15.006778Z","shell.execute_reply":"2025-06-16T09:43:20.531317Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!apt-get update\n!apt-get install -y libboost-all-dev","metadata":{"_uuid":"986263d8-c169-4516-9771-c9d2eab1c8e7","_cell_guid":"b8d6e50b-ad78-4693-b7f6-64c7b058d0b5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:45:55.516475Z","iopub.execute_input":"2025-06-16T09:45:55.516774Z","iopub.status.idle":"2025-06-16T09:46:58.763368Z","shell.execute_reply.started":"2025-06-16T09:45:55.516746Z","shell.execute_reply":"2025-06-16T09:46:58.757628Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sửa allocator của concurrent_hash_map cho tương thích oneAPI TBB\n!sed -i 's/std::pair/std::pair/g' src/queue.hpp","metadata":{"_uuid":"3e0103d5-82f5-42d6-9a79-5d10a50dc64b","_cell_guid":"21463b02-2e8f-4ee6-b0cd-a71991da099d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:52:10.084656Z","iopub.execute_input":"2025-06-16T09:52:10.085036Z","iopub.status.idle":"2025-06-16T09:52:10.212841Z","shell.execute_reply.started":"2025-06-16T09:52:10.085003Z","shell.execute_reply":"2025-06-16T09:52:10.207879Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i 's/tbb::concurrent_hash_map< message_type \\*, bool, MembershipKeyHashCompare, *tbb::scalable_allocator>>/tbb::concurrent_hash_map< const message_type \\*, bool, MembershipKeyHashCompare, tbb::scalable_allocator>>/g' src/queue.hpp","metadata":{"_uuid":"937a0ba4-d008-4006-b844-c6f85d9bf51a","_cell_guid":"7de7f2de-76e6-4c71-a1fc-33bdb701a7be","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:53:31.332175Z","iopub.execute_input":"2025-06-16T09:53:31.332496Z","iopub.status.idle":"2025-06-16T09:53:31.461202Z","shell.execute_reply.started":"2025-06-16T09:53:31.332464Z","shell.execute_reply":"2025-06-16T09:53:31.456465Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i 's/typedef tbb::concurrent_hash_map< message_type \\*, bool, MembershipKeyHashCompare, *tbb::scalable_allocator>>/typedef tbb::concurrent_hash_map< const message_type \\*, bool, MembershipKeyHashCompare, tbb::scalable_allocator>>/g' src/queue.hpp","metadata":{"_uuid":"0496d226-7baa-450d-9287-9986f0e67af7","_cell_guid":"9f786c6c-0b8e-438c-977b-eebd6959b146","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:54:52.068444Z","iopub.execute_input":"2025-06-16T09:54:52.068806Z","iopub.status.idle":"2025-06-16T09:54:52.198773Z","shell.execute_reply.started":"2025-06-16T09:54:52.068755Z","shell.execute_reply":"2025-06-16T09:54:52.193473Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i 's/typedef tbb::concurrent_hash_map< message_type \\*, bool, MembershipKeyHashCompare, *tbb::scalable_allocator>>/typedef tbb::concurrent_hash_map< const message_type \\*, bool, MembershipKeyHashCompare, tbb::scalable_allocator>>/g' src/queue.hpp","metadata":{"_uuid":"53da8fc1-ff14-4808-be2e-329bf28302d2","_cell_guid":"366cf81f-d656-48a3-adaa-328eef1e058c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:57:02.05279Z","iopub.execute_input":"2025-06-16T09:57:02.053168Z","iopub.status.idle":"2025-06-16T09:57:02.180982Z","shell.execute_reply.started":"2025-06-16T09:57:02.053137Z","shell.execute_reply":"2025-06-16T09:57:02.176038Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i 's/tbb::concurrent_hash_map< *message_type \\*,/tbb::concurrent_hash_map< const message_type \\*,/g' src/queue.hpp","metadata":{"_uuid":"616b8958-4aeb-4e3f-b271-bc85a76675a4","_cell_guid":"c9ae30f6-682a-4b37-aa42-0fba2269981c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:58:29.74626Z","iopub.execute_input":"2025-06-16T09:58:29.746591Z","iopub.status.idle":"2025-06-16T09:58:29.875783Z","shell.execute_reply.started":"2025-06-16T09:58:29.746558Z","shell.execute_reply":"2025-06-16T09:58:29.869275Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i 's/static size_t hash(message_type \\* message)/static size_t hash(const message_type * message)/g' src/queue.hpp\n!sed -i 's/static bool equal(message_type \\* left, message_type \\* right)/static bool equal(const message_type * left, const message_type * right)/g' src/queue.hpp","metadata":{"_uuid":"6675aaed-f3d7-4b6d-8dc4-e3030dd8c7cd","_cell_guid":"68b0f2ba-866c-4bac-bc1c-e3a8dfbe2de0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:02:57.687713Z","iopub.execute_input":"2025-06-16T10:02:57.688122Z","iopub.status.idle":"2025-06-16T10:02:57.935485Z","shell.execute_reply.started":"2025-06-16T10:02:57.688083Z","shell.execute_reply":"2025-06-16T10:02:57.928923Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Thay thế toàn bộ nội dung hàm equal trong MembershipKeyHashCompare bằng so sánh đơn giản\n!sed -i '/static bool equal(const message_type \\* left, const message_type \\* right)/,/{/c\\ static bool equal(const message_type * left, const message_type * right) { return (*left) == (*right); }' src/queue.hpp","metadata":{"_uuid":"1c3ec9ac-9a71-4d52-8fe2-621870732b5d","_cell_guid":"e3ae04d4-a6da-4e3e-9032-7cd0290862d6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:06:10.43135Z","iopub.execute_input":"2025-06-16T10:06:10.431699Z","iopub.status.idle":"2025-06-16T10:06:10.56052Z","shell.execute_reply.started":"2025-06-16T10:06:10.431666Z","shell.execute_reply":"2025-06-16T10:06:10.555728Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i '/static bool equal(const message_type \\* left, const message_type \\* right)/{N;N;N;N;N;N;N;N;N;s/static bool equal(const message_type \\* left, const message_type \\* right)[^{]*{[^}]*}/static bool equal(const message_type * left, const message_type * right) { return (*left) == (*right); }/}' src/queue.hpp","metadata":{"_uuid":"43f48d35-7097-4d83-8a6f-af42c56caac1","_cell_guid":"dfc0c58f-f70b-4758-bea7-61a5bdca142b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:08:28.290161Z","iopub.execute_input":"2025-06-16T10:08:28.290473Z","iopub.status.idle":"2025-06-16T10:08:28.419014Z","shell.execute_reply.started":"2025-06-16T10:08:28.29044Z","shell.execute_reply":"2025-06-16T10:08:28.413298Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i '/static bool equal(const message_type \\* left, const message_type \\* right)/c\\ static bool equal(const message_type * left, const message_type * right) { return (*left) == (*right); }' src/queue.hpp\n!sed -i '/left -> features.bit_or/d' src/queue.hpp\n!sed -i '/right -> features.bit_or/d' src/queue.hpp\n!sed -i '/left -> signs.bit_or/d' src/queue.hpp\n!sed -i '/right -> signs.bit_or/d' src/queue.hpp\n!sed -i '/left -> scope/d' src/queue.hpp\n!sed -i '/right -> scope/d' src/queue.hpp\n!sed -i '/return true;/d' src/queue.hpp\n!sed -i '/} else {/d' src/queue.hpp\n!sed -i '/return false;/d' src/queue.hpp","metadata":{"_uuid":"fc9e2b1f-a9ed-42ca-baec-b9e4117f561d","_cell_guid":"61585c01-e6e7-42b4-afa5-333a6af27f37","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:10:29.235143Z","iopub.execute_input":"2025-06-16T10:10:29.235485Z","iopub.status.idle":"2025-06-16T10:10:30.406965Z","shell.execute_reply.started":"2025-06-16T10:10:29.235448Z","shell.execute_reply":"2025-06-16T10:10:30.401202Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i '/static bool equal(const message_type \\* left, const message_type \\* right) { return (\\*left) == (\\*right); }/,+10d' src/queue.hpp","metadata":{"_uuid":"2509a96b-e723-453f-a322-300f61ea39e7","_cell_guid":"e75f135b-9f1c-45d8-b58d-4187f06b0e51","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:11:43.246623Z","iopub.execute_input":"2025-06-16T10:11:43.24704Z","iopub.status.idle":"2025-06-16T10:11:43.372994Z","shell.execute_reply.started":"2025-06-16T10:11:43.246997Z","shell.execute_reply":"2025-06-16T10:11:43.368467Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i '/struct MembershipKeyHashCompare {/,/};/c\\struct MembershipKeyHashCompare {\\n static size_t hash(const message_type * message) { return message->hash(); }\\n static bool equal(const message_type * left, const message_type * right) { return (*left) == (*right); }\\n};' src/queue.hpp","metadata":{"_uuid":"be73e71e-3358-4ae4-818e-173b20ec8ce0","_cell_guid":"a5d9f076-6193-450c-a609-7f05605f59e9","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:13:08.730853Z","iopub.execute_input":"2025-06-16T10:13:08.731163Z","iopub.status.idle":"2025-06-16T10:13:08.857634Z","shell.execute_reply.started":"2025-06-16T10:13:08.731133Z","shell.execute_reply":"2025-06-16T10:13:08.853123Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i '1i #include \"queue.hpp\"' src/state.hpp","metadata":{"_uuid":"04811798-1405-49c5-ba4e-a9e70fa3dfeb","_cell_guid":"8836caca-5749-4532-9b7b-3bf96ab55957","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:14:43.781909Z","iopub.execute_input":"2025-06-16T10:14:43.782234Z","iopub.status.idle":"2025-06-16T10:14:43.908554Z","shell.execute_reply.started":"2025-06-16T10:14:43.782205Z","shell.execute_reply":"2025-06-16T10:14:43.903628Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!sed -i '1i #include \"queue.hpp\"' src/state.hpp","metadata":{"_uuid":"b063a3f1-1f3a-4f1b-898a-0f42d95bb8b3","_cell_guid":"34e79924-288d-4941-a991-b58500528710","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:16:27.338168Z","iopub.execute_input":"2025-06-16T10:16:27.338523Z","iopub.status.idle":"2025-06-16T10:16:27.464241Z","shell.execute_reply.started":"2025-06-16T10:16:27.338491Z","shell.execute_reply":"2025-06-16T10:16:27.459691Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!truncate -s 0 src/queue.hpp","metadata":{"_uuid":"0a750b62-d92f-4876-8c1c-95a23c456afa","_cell_guid":"9e043192-f796-43d6-a557-4274dd550adb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:20:07.240714Z","iopub.execute_input":"2025-06-16T10:20:07.241085Z","iopub.status.idle":"2025-06-16T10:20:07.370135Z","shell.execute_reply.started":"2025-06-16T10:20:07.241051Z","shell.execute_reply":"2025-06-16T10:20:07.364648Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"src/queue.hpp\", \"w\") as f:\n f.write(\"\"\"#ifndef QUEUE_H\n#define QUEUE_H\n\n#include \n#include \n#include \n\n#include \n#include \n#include \n#include \n\n#include \"bitmask.hpp\"\n#include \"configuration.hpp\"\n#include \"message.hpp\"\n\ntypedef Message message_type;\n\nclass PriorityKeyComparator {\npublic:\n bool operator()(message_type const * left, message_type const * right) {\n return (*left) < (*right);\n }\n};\n\nstruct MembershipKeyHashCompare {\n static size_t hash(const message_type * message) { return message->hash(); }\n static bool equal(const message_type * left, const message_type * right) { return (*left) == (*right); }\n};\n\ntypedef tbb::concurrent_priority_queue> queue_type;\n\ntypedef tbb::concurrent_hash_map>> membership_table_type;\n\nclass Queue {\npublic:\n Queue(void);\n ~Queue(void);\n\n bool push(Message const & message);\n bool empty(void) const;\n unsigned int size(void) const;\n bool pop(Message & message);\n\nprivate:\n membership_table_type membership;\n queue_type queue;\n};\n\n#endif\n\"\"\")","metadata":{"_uuid":"1ac5b058-76ab-409c-9912-696bef473cca","_cell_guid":"a1e28f2e-5bed-453b-8ac3-e783f100f29b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:20:15.103352Z","iopub.execute_input":"2025-06-16T10:20:15.103656Z","iopub.status.idle":"2025-06-16T10:20:15.115947Z","shell.execute_reply.started":"2025-06-16T10:20:15.103624Z","shell.execute_reply":"2025-06-16T10:20:15.110906Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install .","metadata":{"_uuid":"9ce83b15-f274-4df8-b485-881dff3fa28f","_cell_guid":"feb5d0cb-af54-4e98-9c5c-6abdb040637a","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:20:16.382679Z","iopub.execute_input":"2025-06-16T10:20:16.382961Z","iopub.status.idle":"2025-06-16T10:20:51.042066Z","shell.execute_reply.started":"2025-06-16T10:20:16.382936Z","shell.execute_reply":"2025-06-16T10:20:51.037492Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from gosdt import GOSDT","metadata":{"_uuid":"92ab5e2c-6211-4b9f-9d4f-7070aaa3ac7a","_cell_guid":"d3184cca-0b84-4557-9e22-1a23b69d3e8b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T10:15:18.976014Z","iopub.execute_input":"2025-06-16T10:15:18.97625Z","iopub.status.idle":"2025-06-16T10:15:19.006327Z","shell.execute_reply.started":"2025-06-16T10:15:18.976223Z","shell.execute_reply":"2025-06-16T10:15:19.00073Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom model.gosdt import GOSDT\n\ndataframe = pd.DataFrame(pd.read_csv(\"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/experiments/datasets/iris/data.csv\"))\n\nX = dataframe[dataframe.columns[:-1]]\ny = dataframe[dataframe.columns[-1:]]\n\nhyperparameters = {\n \"regularization\": 0.04,\n \"time_limit\": 3600,\n \"verbose\": True\n}\n\nmodel = GOSDT(hyperparameters)\nmodel.fit(X, y)\n# model.load(\"python/model/model.json\")\n# model.load(\"../gosdt_icml/model.json\")\nprint(\"Execution Time: {}\".format(model.time))\n\nprediction = model.predict(X)\ntraining_accuracy = model.score(X, y)\nprint(\"Training Accuracy: {}\".format(training_accuracy))\nprint(\"Size: {}\".format(model.leaves()))\nprint(\"Loss: {}\".format(1 - training_accuracy))\nprint(\"Risk: {}\".format(\n model.leaves() * hyperparameters[\"regularization\"]\n + 1 - training_accuracy))\nmodel.tree.__initialize_training_loss__(X, y)\nprint(model.tree)\nprint(model.latex())","metadata":{"_uuid":"8779635a-8f5e-4902-884f-feff493e27b5","_cell_guid":"105aa322-830b-41d7-a4d7-65f7b0eaef87","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:14:30.119497Z","iopub.execute_input":"2025-06-16T09:14:30.119897Z","iopub.status.idle":"2025-06-16T09:14:30.17972Z","shell.execute_reply.started":"2025-06-16T09:14:30.119864Z","shell.execute_reply":"2025-06-16T09:14:30.175392Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/build/lib.linux-x86_64-cpython-312\"))\nprint(os.listdir(\"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/python/model\"))","metadata":{"_uuid":"a57097c5-a0d9-41af-9c44-04828174f013","_cell_guid":"270addf0-e589-4bbc-a166-bff147b14127","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:10:49.899314Z","iopub.execute_input":"2025-06-16T09:10:49.899621Z","iopub.status.idle":"2025-06-16T09:10:49.911681Z","shell.execute_reply.started":"2025-06-16T09:10:49.899596Z","shell.execute_reply":"2025-06-16T09:10:49.906384Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\nshutil.copy(\n \"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/build/lib.linux-x86_64-cpython-312/gosdt.cpython-312-x86_64-linux-gnu.so\",\n \"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/python/model/gosdt.so\"\n)","metadata":{"_uuid":"55f5eda8-15ea-4672-87a5-a8035d205663","_cell_guid":"ae830b70-b73a-4d8a-8b59-b93c153c7c5e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:13:55.023466Z","iopub.execute_input":"2025-06-16T09:13:55.023756Z","iopub.status.idle":"2025-06-16T09:13:55.040621Z","shell.execute_reply.started":"2025-06-16T09:13:55.023732Z","shell.execute_reply":"2025-06-16T09:13:55.035668Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/working/GeneralizedOptimalSparseDecisionTrees-master/python/model\")","metadata":{"_uuid":"1564dc5a-804c-4698-88d3-04e86792ad45","_cell_guid":"a42dc274-3641-4047-bcd0-d9ee284dec46","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:14:10.052427Z","iopub.execute_input":"2025-06-16T09:14:10.052728Z","iopub.status.idle":"2025-06-16T09:14:10.064226Z","shell.execute_reply.started":"2025-06-16T09:14:10.052704Z","shell.execute_reply":"2025-06-16T09:14:10.057526Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nprint(sys.version)","metadata":{"_uuid":"cc707d31-6d5b-4230-b0a2-57903c5df06d","_cell_guid":"eb405602-f0f2-4a45-b129-933cc157ebaa","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-16T09:12:56.956103Z","iopub.execute_input":"2025-06-16T09:12:56.956437Z","iopub.status.idle":"2025-06-16T09:12:56.966376Z","shell.execute_reply.started":"2025-06-16T09:12:56.95641Z","shell.execute_reply":"2025-06-16T09:12:56.960733Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"c9b61d7c-48d8-49e1-ae58-c49a85b25cbe","_cell_guid":"a534d479-6ba5-4c67-b20f-a3e59bc28e96","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"e25407cd-4666-4f8f-b3f1-1129d1874743","_cell_guid":"33dd3ecd-fbe5-4982-ac06-0bc60725b256","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"_uuid":"0aeb3579-bd8e-4507-a325-8d4663f7a205","_cell_guid":"bd3d2a15-9c69-4ccd-8958-fc4f14882a81","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"!pip install gosdt","metadata":{"_uuid":"ccea3a5f-6245-4d42-a06d-c96b542fbab4","_cell_guid":"77e84369-f20e-41c5-b776-5033ab525791","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-15T14:42:59.202642Z","iopub.execute_input":"2025-06-15T14:42:59.202973Z","iopub.status.idle":"2025-06-15T14:43:04.082022Z","shell.execute_reply.started":"2025-06-15T14:42:59.202945Z","shell.execute_reply":"2025-06-15T14:43:04.078655Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom gosdt import GOSDTClassifier\n\nprint(dir(GOSDTClassifier))","metadata":{"_uuid":"a14836c3-2399-425c-bd89-b03db302a425","_cell_guid":"9f6d17ad-98ff-4a2b-82ba-3b3647261a91","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-15T14:46:12.067847Z","iopub.execute_input":"2025-06-15T14:46:12.068166Z","iopub.status.idle":"2025-06-15T14:46:12.079805Z","shell.execute_reply.started":"2025-06-15T14:46:12.068142Z","shell.execute_reply":"2025-06-15T14:46:12.072966Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"help(GOSDTClassifier)","metadata":{"_uuid":"7934740d-f784-4549-8e59-085e8432d3a2","_cell_guid":"889dbb6d-6bef-4666-882a-b5b224a7e0d3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-15T14:46:20.919695Z","iopub.execute_input":"2025-06-15T14:46:20.919981Z","iopub.status.idle":"2025-06-15T14:46:20.939452Z","shell.execute_reply.started":"2025-06-15T14:46:20.919957Z","shell.execute_reply":"2025-06-15T14:46:20.935255Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import inspect\nfrom gosdt import GOSDTClassifier\n\n# In mã nguồn toàn bộ class\nprint(inspect.getsource(GOSDTClassifier))","metadata":{"_uuid":"9118e6c1-6e80-4726-9842-316640ef6538","_cell_guid":"bbd0c222-67ab-4a40-9e36-a619a65c8be2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-15T14:46:41.846382Z","iopub.execute_input":"2025-06-15T14:46:41.846673Z","iopub.status.idle":"2025-06-15T14:46:41.869378Z","shell.execute_reply.started":"2025-06-15T14:46:41.846649Z","shell.execute_reply":"2025-06-15T14:46:41.864647Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"800525f2-d59d-405d-91fc-443ebf7b8150","_cell_guid":"df2b1231-af22-4c32-acb2-909e53b8d02c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"9157be20-2fa9-4207-8154-a03d51dd0e3a","_cell_guid":"192bf42f-4f65-4a71-9697-d25cc46fb42f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"a496707b-2b31-4511-aae4-e12a73018fa4","_cell_guid":"fdc322b4-702f-4f33-a18a-24bf61acad2c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"d6885889-13fc-46a7-9a42-8936b63eb738","_cell_guid":"392d60f4-4bcc-4b8d-b523-a2a95cd534f5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"63bb581a-7162-4330-a064-1c098599f0a6","_cell_guid":"3daf83af-5620-407c-91e3-1092fc1c4849","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import train_test_split\nfrom gosdt import ThresholdGuessBinarizer, GOSDTClassifier\nimport pickle\nfrom sklearn.metrics import confusion_matrix, classification_report\n\n# 1. CẤU HÌNH\nGBDT_N_EST = 50\nGBDT_MAX_DEPTH = 2\nREGULARIZATION = 0.0005\nSIMILAR_SUPPORT = True\nDEPTH_BUDGET = 8\nTIME_LIMIT = 1800 # 30 phút\nVERBOSE = 2\nRANDOM_STATE = 42\n\n# 2. ĐỌC DỮ LIỆU (THAY BẰNG CÁCH ĐỌC CỦA BẠN)\n# train_merged = pl.read_csv(\"path/to/your/data.csv\")\n\n# 3. HÀM CHIA DỮ LIỆU POLARS\ndef polars_train_test_split(\n df: pl.DataFrame, \n target_col: str, \n test_size: float = 0.2, \n random_state: int = RANDOM_STATE\n):\n # Lấy indices và target values\n indices = np.arange(len(df))\n y = df[target_col].to_numpy()\n \n # Chia indices dùng sklearn (stratify)\n train_idx, test_idx = train_test_split(\n indices,\n test_size=test_size,\n stratify=y,\n random_state=random_state\n )\n \n # Tạo mask dạng Polars Series\n train_mask = pl.Series(\"indices\", train_idx)\n \n # Chia DataFrame\n train_df = df.filter(pl.arange(0, df.height).is_in(train_mask))\n test_df = df.filter(~pl.arange(0, df.height).is_in(train_mask))\n \n return (\n train_df.drop(target_col), # X_train\n train_df.select(target_col), # y_train\n test_df.drop(target_col), # X_test\n test_df.select(target_col) # y_test\n )\n\n# 4. CHIA DỮ LIỆU\nX_train, y_train, X_test, y_test = polars_train_test_split(\n train_merged,\n target_col=\"target\",\n test_size=0.2\n)\n\n# 5. CHUYỂN SANG NUMPY CHO GOSDT\ndef to_numpy(df: pl.DataFrame):\n return df.to_numpy().astype(np.float32) # Giảm bộ nhớ\n\nX_train_np = to_numpy(X_train)\ny_train_np = y_train.to_numpy().ravel() # 1D array\nX_test_np = to_numpy(X_test)\ny_test_np = y_test.to_numpy().ravel()\n\n# 6. GUESS THRESHOLDS\nprint(\"=== GUESSING THRESHOLDS ===\")\nenc = ThresholdGuessBinarizer(\n n_estimators=GBDT_N_EST,\n max_depth=GBDT_MAX_DEPTH,\n random_state=RANDOM_STATE,\n n_jobs=2\n)\nX_train_guessed = enc.fit_transform(X_train_np, y_train_np)\nX_test_guessed = enc.transform(X_test_np)\n\n# 7. WARM START\nprint(\"=== WARM START ===\")\ngbdt = GradientBoostingClassifier(\n n_estimators=GBDT_N_EST,\n max_depth=GBDT_MAX_DEPTH,\n random_state=RANDOM_STATE\n)\ngbdt.fit(X_train_guessed, y_train_np)\nwarm_labels = gbdt.predict(X_train_guessed)\n\n# 8. TRAIN GOSDT\nprint(\"=== TRAINING GOSDT ===\")\nclf = GOSDTClassifier(\n regularization=REGULARIZATION,\n similar_support=SIMILAR_SUPPORT,\n depth_budget=DEPTH_BUDGET,\n time_limit=TIME_LIMIT,\n verbose=VERBOSE,\n worker_limit=2,\n balance=True,\n look_ahead=True,\n upperbound_guess=0.9\n)\n\ntry:\n clf.fit(X_train_guessed, y_train_np, y_ref=warm_labels)\nexcept Exception as e:\n print(f\"Training error: {str(e)}\")\n\n# 9. ĐÁNH GIÁ\nif hasattr(clf, 'result_'):\n print(\"\\n=== KẾT QUẢ ===\")\n print(f\"Training time: {clf.result_.time:.1f}s\")\n print(f\"Tree depth: {clf.result_.depth}\")\n print(f\"Leaves: {clf.result_.leaves}\")\n \n # Độ chính xác\n train_acc = clf.score(X_train_guessed, y_train_np)\n test_acc = clf.score(X_test_guessed, y_test_np)\n print(f\"\\nTrain Accuracy: {train_acc:.4f}\")\n print(f\"Test Accuracy: {test_acc:.4f}\")\n \n # Classification report\n y_pred = clf.predict(X_test_guessed)\n print(\"\\nClassification Report:\")\n print(classification_report(y_test_np, y_pred))\n \n # Confusion matrix\n print(\"\\nConfusion Matrix:\")\n print(confusion_matrix(y_test_np, y_pred))\n \n # Hiển thị cây\n print(\"\\nDECISION TREE:\")\n print(clf.get_result().optimal_model)\n \n # Lưu model\n with open('gosdt_model.pkl', 'wb') as f:\n pickle.dump(clf, f)\n print(\"\\nModel saved to 'gosdt_model.pkl'\")\nelse:\n print(\"Training failed - No results available\")","metadata":{"_uuid":"60a86d5a-7eae-49ff-96e2-c80f1f67ea34","_cell_guid":"ad5e2f1a-091b-4b24-9d06-ca55341ebc60","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-06-15T10:42:24.059448Z","iopub.execute_input":"2025-06-15T10:42:24.059836Z","iopub.status.idle":"2025-06-15T10:43:20.820161Z","shell.execute_reply.started":"2025-06-15T10:42:24.059804Z","shell.execute_reply":"2025-06-15T10:43:20.813935Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"4bc3f0cf-236a-44cd-818e-23e88bce10c0","_cell_guid":"240e7917-9011-42c7-89a4-33b579b80f87","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"18dc9a7b-9ca9-459b-9d9e-82120d7ecb3a","_cell_guid":"acae14de-8b16-4bbf-8887-001910c1842b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"998048e6-b341-4f54-898a-800fd80549ad","_cell_guid":"ad6fb6b6-c370-438d-a668-36a76334fb99","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"6d82926f-bd90-4a7a-88fd-e109ea386542","_cell_guid":"41cc1d42-4d5b-477d-bb09-faa02d9664e2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"a39c9a39-157f-499e-a451-06b6cd59b166","_cell_guid":"6e210901-3fc5-4509-a9d9-c3c4b8c698e3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"f4cb4a31-18b6-4ff7-b5cf-26caf6bb8ed7","_cell_guid":"53b703f7-d0c2-4d5e-b7d9-43e05be62767","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"7d67edfd-2029-4f79-a7be-03462cb0d8b4","_cell_guid":"6766e880-149d-46e4-a0c2-bb4366aeb203","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom gosdt import GOSDTClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# 1. Đọc dữ liệu\ndata = pd.read_csv(\"du_lieu_cua_ban.csv\")\n\n# 2. Tách biến mục tiêu\nTARGET_COL = 'target' # Thay bằng tên cột mục tiêu của bạn\ny = data[TARGET_COL]\nX = data.drop(TARGET_COL, axis=1)\n\n# 3. Xử lý giá trị null\ndef prepare_data_with_null(df):\n \"\"\"Xử lý null và chuyển đổi sang boolean\"\"\"\n processed = df.copy()\n for col in processed.columns:\n processed[f'{col}_isnull'] = processed[col].isnull()\n processed = processed.fillna(0)\n return processed.astype(bool)\n\nX_processed = prepare_data_with_null(X)\n\n# 4. Chia tập train/test PHÂN TẦNG\nX_train, X_test, y_train, y_test = train_test_split(\n X_processed,\n y,\n test_size=0.2,\n random_state=42,\n stratify=y # QUAN TRỌNG: Phân tầng theo biến mục tiêu\n)\n\n# 5. Kiểm tra phân phối\nprint(\"\\nPhân phối biến mục tiêu:\")\nprint(f\"Toàn bộ: {y.value_counts(normalize=True)}\")\nprint(f\"Train: {y_train.value_counts(normalize=True)}\")\nprint(f\"Test: {y_test.value_counts(normalize=True)}\")\n\n# 6. Thiết lập và huấn luyện mô hình GOSDT\nmodel = GOSDTClassifier(\n regularization=0.001,\n balance=True, # Bật xử lý imbalance\n depth_budget=None,\n time_limit=600,\n worker_limit=4,\n verbose=True,\n rule_list=False\n)\n\nprint(\"\\nBắt đầu huấn luyện GOSDT với xử lý imbalance...\")\nmodel.fit(X_train, y_train)\nprint(\"Huấn luyện hoàn tất!\")\n\n# 7. Đánh giá chi tiết\ndef evaluate_model(model, X, y, dataset_name):\n y_pred = model.predict(X)\n print(f\"\\nĐÁNH GIÁ TRÊN {dataset_name.upper()}:\")\n print(classification_report(y, y_pred))\n \n # Confusion matrix\n cm = confusion_matrix(y, y_pred)\n disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n disp.plot(cmap='Blues')\n plt.title(f'Confusion Matrix - {dataset_name}')\n plt.show()\n \n # Độ chính xác\n accuracy = model.score(X, y)\n print(f\"Độ chính xác: {accuracy:.2%}\")\n\n# Đánh giá trên tập train và test\nevaluate_model(model, X_train, y_train, \"TRAIN\")\nif len(X_test) > 0:\n evaluate_model(model, X_test, y_test, \"TEST\")\n\n# 8. Hiển thị cây quyết định\nprint(\"\\nCẤU TRÚC CÂY QUYẾT ĐỊNH HOÀN CHỈNH:\")\nprint(model.get_result().optimal_model)\n\n# 9. Lưu mô hình và kết quả\nimport pickle\nimport datetime\n\ntimestamp = datetime.datetime.now().strftime(\"%Y%m%d_%H%M%S\")\nmodel_name = f\"gosdt_model_imbalanced_{timestamp}.pkl\"\n\nwith open(model_name, 'wb') as f:\n pickle.dump(model, f)\nprint(f\"\\nĐã lưu mô hình tại: {model_name}\")\n\n# 10. Lưu kết quả phân tích\nreport_name = f\"model_report_{timestamp}.txt\"\nwith open(report_name, 'w') as f:\n f.write(\"KẾT QUẢ MÔ HÌNH GOSDT\\n\")\n f.write(\"====================\\n\\n\")\n f.write(f\"Thời gian huấn luyện: {timestamp}\\n\")\n f.write(f\"Số lượng mẫu huấn luyện: {len(X_train)}\\n\")\n f.write(f\"Số lượng mẫu kiểm tra: {len(X_test)}\\n\\n\")\n f.write(\"CẤU TRÚC CÂY:\\n\")\n f.write(str(model.get_result().optimal_model))\n \nprint(f\"Đã lưu báo cáo tại: {report_name}\")","metadata":{"_uuid":"642c9e46-1e33-499c-8715-ef094144cd72","_cell_guid":"06b05db0-03f7-44e7-afcb-bbd30038d796","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"3f9d51ef-c5d2-431c-82d8-1985f518a9f2","_cell_guid":"3633f8f9-8315-4a13-9fec-b4e2af2e5c40","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"7331a7e2-3fb5-4eae-91e7-64571b28703b","_cell_guid":"75cbccc0-5648-4894-a800-7a18ed7726ce","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"fb69c7b3-cb2b-4423-98f3-cb929aee97be","_cell_guid":"f08788db-04d5-4bee-a251-8c664cc462fd","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"abd44500-1e83-4afb-ae42-f3275880722a","_cell_guid":"ae27c46a-7d12-499a-a0a5-c54729c03d0c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"619f0519-faae-4a41-8360-9e32c46f5c67","_cell_guid":"bae30819-e57e-465c-b37d-9ba6cd18390d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}