{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665},{"sourceType":"datasetVersion","sourceId":14322821,"datasetId":9139823,"databundleVersionId":15128912,"isSourceIdPinned":true},{"sourceType":"datasetVersion","sourceId":14401390,"datasetId":9197700,"databundleVersionId":15216079},{"sourceType":"kernelVersion","sourceId":290230831},{"sourceType":"kernelVersion","sourceId":290326357},{"sourceType":"kernelVersion","sourceId":290374285}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, glob, json\nimport numpy as np\nimport xgboost as xgb\nfrom scipy import sparse\nfrom sklearn.model_selection import train_test_split\n\n# --- 1. CONFIG PATHS ---\nFEAT_ROOT = \"/kaggle/input/malware-features-ckpt-v5/features_full_v4\"\nY_TRAIN_NPY = os.path.join(FEAT_ROOT, \"y_train.npy\")\nXTR_NUM_NPY = os.path.join(FEAT_ROOT, \"Xtr_num.npy\")\n# Tìm folder X_parts\nCAND_XPART_DIRS = [\n    \"/kaggle/working/rf_gini_out/X_parts\",\n    \"/kaggle/input/rf-gini-out/rf_gini_out/X_parts\",\n    \"/kaggle/input/rf_gini_out/rf_gini_out/X_parts\",\n    \"/kaggle/input/**/rf_gini_out/X_parts\",\n]\nX_PART_DIR = None\nfor p in CAND_XPART_DIRS:\n    if \"*\" in p:\n        hits = glob.glob(p, recursive=True)\n        if hits: X_PART_DIR = hits[0]; break\n    elif os.path.isdir(p): X_PART_DIR = p; break\n\n# --- 2. LOAD DATA GỐC (FULL) ---\nprint(\"⏳ Đang load dữ liệu gốc...\")\ny_full = np.load(Y_TRAIN_NPY).astype(np.int64)\nif y_full.min() == 1: y_full = y_full - 1\n\nprint(f\"📊 Tổng số mẫu trong dataset: {len(y_full)} (Phải là ~10868)\")\n\n# Load Features\nxparts = sorted(glob.glob(os.path.join(X_PART_DIR, \"X_part_*.npz\")))\nX_op = sparse.vstack([sparse.load_npz(fp).tocsr() for fp in xparts])\nX_num = sparse.csr_matrix(np.load(XTR_NUM_NPY), dtype=np.float32)\nX_C = sparse.hstack([X_op, X_num], format=\"csr\")\n\n# --- 3. SPLIT LẠI CHÍNH XÁC ---\n# Phải dùng idx của TOÀN BỘ dữ liệu\nidx = np.arange(len(y_full))\ntr_idx, va_idx = train_test_split(\n    idx, \n    test_size=0.20, \n    random_state=20251226, # KHỚP VỚI CNN\n    stratify=y_full\n)\n\nprint(f\"✅ Số lượng mẫu Validation cần dự đoán: {len(va_idx)} (Phải là 2174)\")\n\nX_va = X_C[va_idx]\ny_va = y_full[va_idx] # Labels chuẩn\n\n# --- 4. PREDICT & SAVE ---\nMODEL_PATH = \"/kaggle/working/xgb_out_C/xgb_C_opcode_segment_2nd.json\"\n# Tìm model nếu không thấy\nif not os.path.exists(MODEL_PATH):\n    search = glob.glob(f\"/kaggle/input/**/xgb_C_opcode_segment_2nd.json\", recursive=True)\n    if search: MODEL_PATH = search[0]\n\nprint(f\"🤖 Load model: {MODEL_PATH}\")\nbst = xgb.Booster()\nbst.load_model(MODEL_PATH)\n\ndval = xgb.DMatrix(X_va)\nxgb_probs = bst.predict(dval)\n\nprint(f\"📉 Shape output XGBoost: {xgb_probs.shape}\")\n\nnp.save('xgb_probs.npy', xgb_probs)\nnp.save('y_valid_labels.npy', y_va)\nprint(\"✅ ĐÃ KHỚP! Bạn hãy chạy lại cell Ensemble ngay.\")\n# else:\n# print(\"❌ Vẫn sai số lượng mẫu. Hãy kiểm tra lại file input!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T15:40:20.281095Z","iopub.execute_input":"2026-01-06T15:40:20.281844Z","iopub.status.idle":"2026-01-06T15:40:36.597495Z","shell.execute_reply.started":"2026-01-06T15:40:20.281812Z","shell.execute_reply":"2026-01-06T15:40:36.596354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\nprint(\"⏳ Đang tổng hợp kết quả...\")\n\n# 1. Load 2 file xác suất\ntry:\n    p_xgb = np.load('xgb_probs.npy') # Từ XGBoost\n    p_cnn = np.load('/kaggle/input/cnn-t4-x2/cnn_probs.npy') # Từ CNN vừa chạy xong\n    y_true = np.load('y_valid_labels.npy') # Ground Truth (Nhãn đúng)\nexcept FileNotFoundError as e:\n    print(f\"❌ LỖI: Thiếu file! Chi tiết: {e}\")\n    # Dừng chương trình nếu thiếu file\n    raise\n\n# 2. Kiểm tra khớp lệnh\nprint(f\"Shape XGBoost: {p_xgb.shape}\")\nprint(f\"Shape CNN    : {p_cnn.shape}\")\nprint(f\"Shape Labels : {y_true.shape}\")\n\n# Cắt ngắn nếu lệch 1-2 dòng (do lỗi load ảnh chẳng hạn)\nmin_len = min(len(p_xgb), len(p_cnn), len(y_true))\nif len(p_xgb) != len(p_cnn):\n    print(f\"⚠️ CẢNH BÁO: Số lượng mẫu không khớp! Đang cắt về {min_len} mẫu.\")\n    \np_xgb = p_xgb[:min_len]\np_cnn = p_cnn[:min_len]\ny_true = y_true[:min_len]\n\n# 3. Tính toán Ensemble\n# Công thức: Final = 0.5 * XGB + 0.5 * CNN\n# Bạn có thể chỉnh alpha để xem cái nào tốt hơn (vd: 0.4 hoặc 0.6)\nalpha = 0.5 \np_ensemble = (alpha * p_cnn) + ((1 - alpha) * p_xgb)\ny_pred = np.argmax(p_ensemble, axis=1)\n\n# 4. Xuất báo cáo\nacc = accuracy_score(y_true, y_pred)\nprint(\"=\"*40)\nprint(f\"ENSEMBLE ACCURACY: {acc*100:.4f}%\")\nprint(\"=\"*40)\n\nprint(\"\\n--- Chi tiết từng lớp ---\")\ntarget_names = [\"Ramnit\", \"Lollipop\", \"Kelihos_ver3\", \"Vundo\", \"Simda\", \"Tracur\", \"Kelihos_ver1\", \"Obfuscator.ACY\", \"Gatak\"]\nprint(classification_report(y_true, y_pred, target_names=target_names, digits=4))\n\n# 5. Vẽ Confusion Matrix\ncm = confusion_matrix(y_true, y_pred)\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=target_names, yticklabels=target_names)\nplt.title(f'Ensemble Confusion Matrix (Acc: {acc*100:.2f}%)')\nplt.ylabel('Thực tế (True Label)')\nplt.xlabel('Dự đoán (Predicted Label)')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-06T15:40:36.598485Z","iopub.execute_input":"2026-01-06T15:40:36.598886Z","iopub.status.idle":"2026-01-06T15:40:37.170603Z","shell.execute_reply.started":"2026-01-06T15:40:36.598860Z","shell.execute_reply":"2026-01-06T15:40:37.169759Z"}},"outputs":[],"execution_count":null}]}