{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4117,"databundleVersionId":46665,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14322821,"sourceType":"datasetVersion","datasetId":9139823,"isSourceIdPinned":true},{"sourceId":14401390,"sourceType":"datasetVersion","datasetId":9197700},{"sourceId":14394783,"sourceType":"datasetVersion","datasetId":9193286}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# ONE CELL: XGBoost A/B/C\n#   A = Opcode(filtered) only  (from rf_gini_out/X_parts/*.npz)\n#   B = Opcode(filtered) + Segment (from features_full_v4/Xtr_num.npy)\n#   C = Opcode(filtered) + Segment + 2nd (sf_names) (from Xtr_num.npy)\n#\n# Needs:\n#   /kaggle/input/rf-gini-out/rf_gini_out/X_parts/X_part_*.npz   (or /kaggle/working/...)\n#   /kaggle/input/malware-features-ckpt-v5/features_full_v4/\n#       train_ids.txt, y_train.npy, Xtr_num.npy, config.json\n#\n# Output:\n#   /kaggle/working/xgb_out_A/*\n#   /kaggle/working/xgb_out_B/*\n#   /kaggle/working/xgb_out_C/*\n# ============================================================\nimport os, glob, json, time\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, log_loss, classification_report\nimport xgboost as xgb\n\n# -----------------------\n# Paths\n# -----------------------\nFEAT_ROOT = \"/kaggle/input/malware-features-ckpt-v5/features_full_v4\"\nTRAIN_IDS_TXT = os.path.join(FEAT_ROOT, \"train_ids.txt\")\nY_TRAIN_NPY   = os.path.join(FEAT_ROOT, \"y_train.npy\")\nXTR_NUM_NPY   = os.path.join(FEAT_ROOT, \"Xtr_num.npy\")\nCFG_JSON      = os.path.join(FEAT_ROOT, \"config.json\")\n\nCAND_XPART_DIRS = [\n    \"/kaggle/working/rf_gini_out/X_parts\",\n    \"/kaggle/input/rf-gini-out/rf_gini_out/X_parts\",\n    \"/kaggle/input/rf_gini_out/rf_gini_out/X_parts\",\n    \"/kaggle/input/**/rf_gini_out/X_parts\",\n]\nX_PART_DIR = None\nfor p in CAND_XPART_DIRS:\n    if \"*\" in p:\n        hits = glob.glob(p, recursive=True)\n        hits = [h for h in hits if os.path.isdir(h)]\n        if hits:\n            X_PART_DIR = hits[0]\n            break\n    else:\n        if os.path.isdir(p):\n            X_PART_DIR = p\n            break\n\nassert X_PART_DIR is not None, \"Không tìm thấy X_parts. Hãy attach dataset rf_gini_out.\"\nassert os.path.exists(TRAIN_IDS_TXT), \"Không thấy train_ids.txt\"\nassert os.path.exists(Y_TRAIN_NPY), \"Không thấy y_train.npy\"\nassert os.path.exists(XTR_NUM_NPY), \"Không thấy Xtr_num.npy\"\nassert os.path.exists(CFG_JSON), \"Không thấy config.json\"\n\nOUT_A = \"/kaggle/working/xgb_out_A\"\nOUT_B = \"/kaggle/working/xgb_out_B\"\nOUT_C = \"/kaggle/working/xgb_out_C\"\nos.makedirs(OUT_A, exist_ok=True)\nos.makedirs(OUT_B, exist_ok=True)\nos.makedirs(OUT_C, exist_ok=True)\n\nprint(\"[OK] X_PART_DIR:\", X_PART_DIR)\nprint(\"[OK] FEAT_ROOT:\", FEAT_ROOT)\n\n# -----------------------\n# Load ids + labels (aligned)\n# -----------------------\nwith open(TRAIN_IDS_TXT, \"r\", encoding=\"utf-8\") as f:\n    train_ids = [x.strip() for x in f if x.strip()]\ny = np.load(Y_TRAIN_NPY).astype(np.int64)\n\n# normalize y to 0..8\nif y.min() == 1 and y.max() == 9:\n    y = y - 1\n\nn_samples = len(train_ids)\nassert len(y) == n_samples, f\"Mismatch: len(y)={len(y)} vs n_samples={n_samples}\"\nn_classes = int(y.max() + 1)\nprint(\"[INFO] n_samples:\", n_samples, \"| n_classes:\", n_classes)\n\n# -----------------------\n# Load opcode filtered X (CSR) from X_parts\n# -----------------------\nxparts = sorted(glob.glob(os.path.join(X_PART_DIR, \"X_part_*.npz\")))\nassert xparts, \"Không có file X_part_*.npz trong X_PART_DIR\"\n\nt0 = time.time()\nmats = []\nfor fp in xparts:\n    Xi = sparse.load_npz(fp).tocsr()\n    mats.append(Xi)\n    print(f\"[LOAD] {os.path.basename(fp)} shape={Xi.shape} nnz={Xi.nnz}\")\nX_op = sparse.vstack(mats, format=\"csr\")\ndel mats\nprint(\"[X_op] shape:\", X_op.shape, \"| nnz:\", X_op.nnz, \"| load_min:\", (time.time()-t0)/60)\nassert X_op.shape[0] == n_samples, f\"Row mismatch: X_op has {X_op.shape[0]} rows but n_samples={n_samples}\"\n\n# -----------------------\n# Load numeric features: segment + 2nd from Xtr_num.npy using config.json\n# -----------------------\ncfg = json.load(open(CFG_JSON, \"r\", encoding=\"utf-8\"))\nsegments = cfg.get(\"segments\", [])\nsf_names = cfg.get(\"sf_names\", [])\n\nn_seg = len(segments)\nn_sf  = len(sf_names)\nassert n_seg > 0 and n_sf > 0, \"config.json thiếu segments hoặc sf_names?\"\n\nX_num = np.load(XTR_NUM_NPY)\nassert X_num.shape[0] == n_samples, f\"Row mismatch: X_num has {X_num.shape[0]} rows but n_samples={n_samples}\"\n\nneed_cols = n_seg + n_sf\nif X_num.shape[1] < need_cols:\n    raise RuntimeError(f\"Xtr_num.npy has {X_num.shape[1]} cols < needed {need_cols} (segments+sf_names).\")\n\n# assume layout: [segments..., sf_names...]\nX_seg = X_num[:, :n_seg]\nX_2nd = X_num[:, n_seg:n_seg+n_sf]\n\nX_seg_sp = sparse.csr_matrix(X_seg, dtype=np.float32)\nX_2nd_sp = sparse.csr_matrix(X_2nd, dtype=np.float32)\n\nprint(\"[X_num] shape:\", X_num.shape)\nprint(\"[SEG] shape:\", X_seg_sp.shape, \"segments=\", n_seg)\nprint(\"[2ND] shape:\", X_2nd_sp.shape, \"sf_names=\", n_sf)\n\n# -----------------------\n# Build A / B / C matrices\n# -----------------------\nX_A = X_op\nX_B = sparse.hstack([X_op, X_seg_sp], format=\"csr\")\nX_C = sparse.hstack([X_op, X_seg_sp, X_2nd_sp], format=\"csr\")\n\nprint(\"[X_A] shape:\", X_A.shape, \"| nnz:\", X_A.nnz)\nprint(\"[X_B] shape:\", X_B.shape, \"| nnz:\", X_B.nnz)\nprint(\"[X_C] shape:\", X_C.shape, \"| nnz:\", X_C.nnz)\n\n# -----------------------\n# Split indices (same split for A/B/C)\n# -----------------------\nidx = np.arange(n_samples)\ntr_idx, va_idx = train_test_split(\n    idx, test_size=0.20, random_state=20251226, stratify=y\n)\ny_tr = y[tr_idx]\ny_va = y[va_idx]\n\n# -----------------------\n# Params (GIỮ NGUYÊN của bạn)\n# -----------------------\n\nparams_A = {\n    \"objective\": \"multi:softprob\",\n    \"num_class\": n_classes,\n    \"eval_metric\": \"mlogloss\",\n    \"tree_method\": \"hist\",\n    \"max_bin\": 256,\n\n    \"eta\": 0.04,\n    \"max_depth\": 2,\n    \"min_child_weight\": 110,   # (A) bớt chặt hơn 130\n    \"gamma\": 24,               # (A) bớt chặt hơn 29\n\n    \"subsample\": 0.42,\n    \"colsample_bytree\": 0.42,\n\n    \"lambda\": 75.0,\n    \"alpha\": 22.0,\n\n    \"seed\": 20251226,\n    \"verbosity\": 1,\n}\n\nparams_B = {\n    \"objective\": \"multi:softprob\",\n    \"num_class\": n_classes,\n    \"eval_metric\": \"mlogloss\",\n    \"tree_method\": \"hist\",\n    \"max_bin\": 256,\n\n    \"eta\": 0.04,\n    \"max_depth\": 2,\n    \"min_child_weight\": 95,    # (B) nới tiếp\n    \"gamma\": 20,\n\n    \"subsample\": 0.48,\n    \"colsample_bytree\": 0.48,\n\n    \"lambda\": 60.0,\n    \"alpha\": 18.0,\n\n    \"seed\": 20251226,\n    \"verbosity\": 1,\n}\n\nparams_C = {\n    \"objective\": \"multi:softprob\",\n    \"num_class\": n_classes,\n    \"eval_metric\": \"mlogloss\",\n    \"tree_method\": \"hist\",\n    \"max_bin\": 256,\n\n    \"eta\": 0.04,\n    \"max_depth\": 3,            # (C) tăng depth để ăn được interactions opcode<->numeric\n    \"min_child_weight\": 60,    # (C) nới rõ để 2nd features có tác dụng\n    \"gamma\": 10,\n\n    \"subsample\": 0.62,         # (C) nhiều dữ liệu/cây hơn\n    \"colsample_bytree\": 0.62,  # (C) nhiều features/cây hơn\n\n    \"lambda\": 35.0,\n    \"alpha\": 10.0,\n\n    \"seed\": 20251226,\n    \"verbosity\": 1,\n}\n\n\nnum_boost_round = 1500\nearly_stopping_rounds = 50\nverbose_eval = 50\n\ndef train_one(tag, X_all, out_dir, params):\n    X_tr = X_all[tr_idx]\n    X_va = X_all[va_idx]\n\n    dtrain = xgb.DMatrix(X_tr, label=y_tr)\n    dval   = xgb.DMatrix(X_va, label=y_va)\n\n    print(f\"\\n===== TRAIN {tag} =====\")\n    t0 = time.time()\n    bst = xgb.train(\n        params=params,\n        dtrain=dtrain,\n        num_boost_round=num_boost_round,\n        evals=[(dval, \"validation\")],\n        early_stopping_rounds=early_stopping_rounds,\n        verbose_eval=verbose_eval\n    )\n    mins = (time.time()-t0)/60\n    print(f\"[{tag}] fit done. minutes:\", mins)\n\n    p_va = bst.predict(dval)\n    pred_va = p_va.argmax(axis=1)\n    acc = accuracy_score(y_va, pred_va)\n    ll  = log_loss(y_va, p_va, labels=list(range(n_classes)))\n\n    print(f\"[{tag}][VAL] accuracy={acc:.4f} | logloss={ll:.4f}\\n\")\n    print(classification_report(y_va, pred_va, digits=4))\n\n    model_path = os.path.join(out_dir, f\"xgb_{tag}.json\")\n    bst.save_model(model_path)\n\n    metrics = {\n        \"tag\": tag,\n        \"val_accuracy\": float(acc),\n        \"val_logloss\": float(ll),\n        \"best_iteration\": int(bst.best_iteration) if bst.best_iteration is not None else None,\n        \"best_score\": float(bst.best_score) if bst.best_score is not None else None,\n        \"n_features\": int(X_all.shape[1]),\n        \"params\": params,\n        \"num_boost_round\": int(num_boost_round),\n        \"early_stopping_rounds\": int(early_stopping_rounds),\n        \"train_minutes\": float(mins),\n    }\n    with open(os.path.join(out_dir, f\"metrics_{tag}.json\"), \"w\", encoding=\"utf-8\") as f:\n        json.dump(metrics, f, indent=2)\n\n    imp = bst.get_score(importance_type=\"gain\")\n    rows = [(int(k[1:]), v) for k, v in imp.items()]\n    pd.DataFrame(rows, columns=[\"feature_index\", \"gain\"]).sort_values(\"gain\", ascending=False)\\\n      .to_csv(os.path.join(out_dir, f\"xgb_gain_{tag}.csv\"), index=False)\n\n    print(f\"[{tag}] saved:\", model_path)\n    return acc, ll\n\naccA, llA = train_one(\"A_opcode_only\", X_A, OUT_A, params_A)\naccB, llB = train_one(\"B_opcode_segment\", X_B, OUT_B, params_B)\naccC, llC = train_one(\"C_opcode_segment_2nd\", X_C, OUT_C, params_C)\n\nprint(\"\\n=== SUMMARY ===\")\nprint(\"A (opcode)                 acc=\", accA, \"logloss=\", llA)\nprint(\"B (opcode+segment)         acc=\", accB, \"logloss=\", llB)\nprint(\"C (opcode+segment+2nd)     acc=\", accC, \"logloss=\", llC)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T20:56:54.997434Z","iopub.execute_input":"2026-01-05T20:56:54.999067Z","iopub.status.idle":"2026-01-05T21:30:19.848516Z","shell.execute_reply.started":"2026-01-05T20:56:54.999019Z","shell.execute_reply":"2026-01-05T21:30:19.847442Z"}},"outputs":[],"execution_count":null}]}