{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665},{"sourceType":"datasetVersion","sourceId":15592323,"datasetId":9960002,"databundleVersionId":16524836},{"sourceType":"kernelVersion","sourceId":309855409}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phase 2+3 — CNN Feature Extraction + XGBoost Training\n\n**Input từ Phase 1:**\n```\n/kaggle/input/notebooks/nhatnguyenhcmusk24/nahhhhhhh/features/\n  ├── image_features.zip      (10,868 × 256×256×3 .npy)\n  ├── tabular_features.csv    (10,868 × 525 opcode features + Class)\n  └── vocabulary.pkl          (top_uni + top_bi)\n```\n\n**Pipeline Phase 2+3:**\n```\nimage_features.zip → extract → EfficientNet-B0 (frozen) → cnn_features (1280-d)\ntabular_features.csv                                     → ngram_features (525-d)\n7z l train.7z + test.7z                                  → metadata (4-d)\ntest.7z → batch extract → Markov + opcode               → test features\n                                                              ↓\nCONCAT → Feature Selection (RF) → XGBoost (Stratified 5-Fold + sample_weight)\n       → Pseudo-labeling → Ensemble → submission.csv\n```","metadata":{}},{"cell_type":"code","source":"import sys, os, gc, time, logging, zipfile, shutil, subprocess, pickle\nimport numpy as np\nimport pandas as pd\nimport scipy.sparse as sp\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nimport torchvision.transforms as T\n\nimport xgboost as xgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.utils.class_weight import compute_sample_weight\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.utils.class_weight import compute_sample_weight\nfrom sklearn.linear_model import LogisticRegression\nimport lightgbm as lgb \nfrom sklearn.metrics import accuracy_score, log_loss\n\n# ── Utils ─────────────────────────────────────────────────────────────────────\nUTILS_PATH = Path('/kaggle/input/datasets/nhatnguyenhcmusk24/logic-feature')\nsys.path.append(str(UTILS_PATH))\nfrom malware_utils import Config, GPU, FileOps, ImageOps, OpcodeOps, CacheOps\n\n# ── Logging ───────────────────────────────────────────────────────────────────\nlogging.basicConfig(\n    level=logging.INFO,\n    format='%(asctime)s [%(levelname)s] %(message)s',\n    datefmt='%H:%M:%S',\n    handlers=[\n        logging.StreamHandler(sys.stdout),\n        logging.FileHandler('/kaggle/working/phase2.log')\n    ]\n)\nlog = logging.getLogger('phase2')\n\n# ── Paths ─────────────────────────────────────────────────────────────────────\nPHASE1_DIR  = Path('/kaggle/input/notebooks/nhatnguyenhcmusk24/data-process-for-microsoft-malware-detection-big/features')\nCOMP_DIR    = Path('/kaggle/input/competitions/malware-classification')\nWORK_DIR    = Path('/kaggle/working')\nIMG_DIR     = WORK_DIR / 'train_images'   # sau khi extract zip\nFEAT_DIR    = WORK_DIR / 'features'\nSCRATCH     = WORK_DIR / 'scratch'\n\nfor d in [IMG_DIR, FEAT_DIR, SCRATCH]:\n    d.mkdir(parents=True, exist_ok=True)\n\n# ── Config ────────────────────────────────────────────────────────────────────\ncfg = Config(base_dir=COMP_DIR)\nGPU.setup()\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nSEED     = 42\nN_FOLDS  = 5\nN_CLASSES= 9\n\nCLASS_NAMES = {\n    1:'Ramnit', 2:'Lollipop', 3:'Kelihos_ver3', 4:'Vundo', 5:'Simda',\n    6:'Tracur', 7:'Kelihos_ver1', 8:'Obfuscator.ACY', 9:'Gatak'\n}\n\nlog.info('Device: %s', DEVICE)\nlog.info('Phase1 dir: %s (exists=%s)', PHASE1_DIR, PHASE1_DIR.exists())\nlog.info('Disk free: %.1f GB', FileOps.get_free_gb())","metadata":{"execution":{"iopub.execute_input":"2026-04-08T14:21:39.495583Z","iopub.status.busy":"2026-04-08T14:21:39.495284Z","iopub.status.idle":"2026-04-08T14:21:39.512252Z","shell.execute_reply":"2026-04-08T14:21:39.511448Z","shell.execute_reply.started":"2026-04-08T14:21:39.495561Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Data process outputs","metadata":{}},{"cell_type":"code","source":"# ── Labels ────────────────────────────────────────────────────────────────────\ndf_labels = pd.read_csv(COMP_DIR / 'trainLabels.csv')\ndf_labels.columns = df_labels.columns.str.strip()\nfor c in df_labels.columns:\n    if c.lower() in ('id','name'): df_labels = df_labels.rename(columns={c:'Id'})\n    if c.lower() == 'class':       df_labels = df_labels.rename(columns={c:'Class'})\n\n# Label offset: XGBoost cần 0-indexed\ndf_labels['label'] = df_labels['Class'] - 1   # 1-9 → 0-8\nALL_IDS = df_labels['Id'].tolist()\nlog.info('Train samples: %d', len(ALL_IDS))\n\n# ── Tabular features từ Phase 1 ───────────────────────────────────────────────\ndf_tab = pd.read_csv(PHASE1_DIR / 'tabular_features.csv', index_col='Id')\nlog.info('Tabular shape: %s', df_tab.shape)\n# Meta data\nMETA_COLS    = ['bytes_size', 'asm_size', 'size_ratio']\ndf_meta  = df_tab[META_COLS].copy()\ndf_tab = df_tab.drop(columns=META_COLS)  \n# Tách Class ra\ny_tab = df_tab.pop('Class') if 'Class' in df_tab.columns else None\n\n# ── Vocabulary ────────────────────────────────────────────────────────────────\ntop_uni, top_bi = CacheOps.load_vocabulary(PHASE1_DIR / 'vocabulary.pkl')\nlog.info('Vocabulary: %d uni + %d bi', len(top_uni), len(top_bi))\n\n# ── Preview ───────────────────────────────────────────────────────────────────\ndisplay(df_tab.head(2))\nprint(f'Tabular shape  : {df_tab.shape}')\nprint(f'Non-zero cols  : {(df_tab > 0).sum(axis=1).describe().round(1)}')","metadata":{"execution":{"iopub.execute_input":"2026-04-08T14:21:39.514090Z","iopub.status.busy":"2026-04-08T14:21:39.513755Z","iopub.status.idle":"2026-04-08T14:21:40.350977Z","shell.execute_reply":"2026-04-08T14:21:40.350207Z","shell.execute_reply.started":"2026-04-08T14:21:39.514067Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Extract image_features.zip → train_images/","metadata":{}},{"cell_type":"code","source":"# ── Extract zip một lần, load nhanh về sau ────────────────────────────────────\nzip_path = PHASE1_DIR / 'image_features.zip'\nalready  = list(IMG_DIR.glob('*.npy'))\n\nif len(already) >= len(ALL_IDS):\n    log.info('Images đã extract: %d files — skip', len(already))\nelse:\n    log.info('Extracting %s → %s ...', zip_path.name, IMG_DIR)\n    t0 = time.time()\n    with zipfile.ZipFile(zip_path, 'r') as zf:\n        zf.extractall(IMG_DIR)\n    n = len(list(IMG_DIR.glob('*.npy')))\n    log.info('Extracted %d files in %.1fs', n, time.time()-t0)\n\nlog.info('Disk free after extract: %.1f GB', FileOps.get_free_gb())","metadata":{"execution":{"iopub.execute_input":"2026-04-08T14:21:40.352273Z","iopub.status.busy":"2026-04-08T14:21:40.351875Z","iopub.status.idle":"2026-04-08T14:21:40.398358Z","shell.execute_reply":"2026-04-08T14:21:40.397763Z","shell.execute_reply.started":"2026-04-08T14:21:40.352246Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CNN Feature Extraction (EfficientNet-B0, Frozen)","metadata":{}},{"cell_type":"code","source":"# ── Dataset ───────────────────────────────────────────────────────────────────\nclass MalwareImageDataset(Dataset):\n    \"\"\"\n    Load .npy tensors (256,256,3) uint8.\n    Normalize về ImageNet mean/std để dùng pretrained weights.\n    \"\"\"\n    def __init__(self, sample_ids: list, img_dir: Path):\n        self.ids     = sample_ids\n        self.img_dir = img_dir\n        self.tf = T.Compose([\n            T.ToTensor(),                          # HWC uint8 → CHW float32 [0,1]\n            T.Normalize(mean=[0.485, 0.456, 0.406],\n                        std =[0.229, 0.224, 0.225]),\n        ])\n\n    def __len__(self):  return len(self.ids)\n\n    def __getitem__(self, idx):\n        arr = np.load(self.img_dir / f'{self.ids[idx]}.npy')  # (256,256,3) uint8\n        return self.tf(arr), self.ids[idx]\n\n\n# ── Model: EfficientNet-B0 — bỏ head, giữ feature extractor ──────────────────\nclass CNNExtractor(nn.Module):\n    def __init__(self):\n        super().__init__()\n        backbone = models.efficientnet_b0(weights='IMAGENET1K_V1')\n        # Bỏ classifier head, giữ features + avgpool\n        self.features = backbone.features\n        self.avgpool  = backbone.avgpool   # AdaptiveAvgPool2d(1)\n        # Freeze toàn bộ\n        for p in self.parameters():\n            p.requires_grad = False\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.avgpool(x)\n        return x.flatten(1)   # → (batch, 1280)\n\n\n# ── Extract features ──────────────────────────────────────────────────────────\nCNN_FEAT_PATH = FEAT_DIR / 'cnn_features_train.npy'\nCNN_IDS_PATH  = FEAT_DIR / 'cnn_ids_train.pkl'\n\nif CNN_FEAT_PATH.exists():\n    log.info('CNN features đã có — skip extraction')\n    cnn_feat_train = np.load(CNN_FEAT_PATH)\n    with open(CNN_IDS_PATH, 'rb') as f:\n        cnn_ids_train = pickle.load(f)\nelse:\n    model = CNNExtractor().to(DEVICE).eval()\n    log.info('EfficientNet-B0 loaded (frozen) — output dim: 1280')\n\n    dataset = MalwareImageDataset(ALL_IDS, IMG_DIR)\n    loader  = DataLoader(dataset, batch_size=64, shuffle=False,\n                         num_workers=4, pin_memory=True)\n\n    all_feats = []\n    all_ids   = []\n    t0 = time.time()\n\n    with torch.no_grad():\n        for imgs, ids in tqdm(loader, desc='CNN extract', ncols=80):\n            feats = model(imgs.to(DEVICE))\n            all_feats.append(feats.cpu().numpy())\n            all_ids.extend(ids)\n\n    cnn_feat_train = np.vstack(all_feats)   # (N, 1280)\n    cnn_ids_train  = all_ids\n\n    np.save(CNN_FEAT_PATH, cnn_feat_train)\n    with open(CNN_IDS_PATH, 'wb') as f:\n        pickle.dump(cnn_ids_train, f, protocol=4)\n\n    elapsed = time.time() - t0\n    log.info('CNN extraction done: shape=%s in %.1f min',\n             cnn_feat_train.shape, elapsed/60)\n\n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n\nprint(f'CNN features shape: {cnn_feat_train.shape}')","metadata":{"execution":{"iopub.execute_input":"2026-04-08T14:21:40.399530Z","iopub.status.busy":"2026-04-08T14:21:40.399246Z","iopub.status.idle":"2026-04-08T14:21:40.434072Z","shell.execute_reply":"2026-04-08T14:21:40.433303Z","shell.execute_reply.started":"2026-04-08T14:21:40.399499Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test set processing\n\nExtract test.7z theo batch → Markov image + Opcode features dùng vocabulary.pkl từ Phase 1","metadata":{}},{"cell_type":"code","source":"TEST_IMG_DIR  = WORK_DIR / 'test_images'\nTEST_CNN_PATH = FEAT_DIR / 'cnn_features_test.npy'\nTEST_TAB_PATH = FEAT_DIR / 'tabular_features_test.csv'\nTEST_META_PATH= FEAT_DIR / 'metadata_test.csv'\nTEST_IMG_DIR.mkdir(exist_ok=True)\n\ndf_sub_template = pd.read_csv(COMP_DIR / 'sampleSubmission.csv')\nTEST_IDS = df_sub_template['Id'].tolist()\nlog.info('Test samples: %d', len(TEST_IDS))\n\n\nfor p in [TEST_CNN_PATH, TEST_TAB_PATH, TEST_META_PATH]:\n    if p.exists():\n        p.unlink()\n        log.info('Deleted stale: %s', p.name)\n        \ntest_done = (TEST_CNN_PATH.exists() and\n             TEST_TAB_PATH.exists() and\n             TEST_META_PATH.exists())\nif test_done:\n    log.info('Test features đã có — skip')\nelse:\n    log.info('Processing test set...')\n\n    # FIX 2: Định nghĩa hàm trước khi dùng\n    def get_file_sizes_by_ext(archive: Path) -> tuple:\n        result = subprocess.run(['7z', 'l', '-ba', str(archive)],\n                                capture_output=True, text=True)\n        bytes_sizes, asm_sizes = {}, {}\n        for line in result.stdout.splitlines():\n            parts = line.strip().split()\n            if len(parts) >= 5:\n                try:\n                    size = int(parts[3])\n                    p    = Path(parts[-1])\n                    if p.suffix == '.bytes': bytes_sizes[p.stem] = size\n                    elif p.suffix == '.asm': asm_sizes[p.stem]   = size\n                except ValueError:\n                    pass\n        return bytes_sizes, asm_sizes\n\n    def process_test_batch(batch_ids, archive, img_dir, raw_records):\n        FileOps.extract_batch(archive, batch_ids, SCRATCH)\n        bytes_idx, asm_idx = {}, {}\n        for p in SCRATCH.rglob('*'):\n            if p.is_file():\n                if p.suffix == '.bytes': bytes_idx[p.stem] = p\n                elif p.suffix == '.asm': asm_idx[p.stem]   = p\n        for sid in batch_ids:\n            bp = bytes_idx.get(sid)\n            ap = asm_idx.get(sid)\n            img = ImageOps.process_image(bp, ap, cfg, exact_entropy=False)\n            np.save(img_dir / f'{sid}.npy', img)\n            counts = OpcodeOps.parse(ap) if ap else {'uni': {}, 'bi': {}}\n            counts['id'] = sid\n            raw_records.append(counts)\n            FileOps.delete_raw(bp, ap)\n        shutil.rmtree(SCRATCH, ignore_errors=True)\n        SCRATCH.mkdir(exist_ok=True)\n\n    # FIX 1: Sort theo archive order\n    log.info('Sorting test IDs by archive order...')\n    test_archive_order = FileOps.get_archive_order(COMP_DIR / 'test.7z')\n    TEST_IDS_SORTED    = FileOps.sort_by_archive_order(TEST_IDS, test_archive_order)\n    test_batches       = list(FileOps.chunked(TEST_IDS_SORTED, cfg.batch_n))\n\n    test_raw_records = []\n    for batch_ids in tqdm(test_batches, desc='Test batches', ncols=80):\n        process_test_batch(batch_ids, COMP_DIR / 'test.7z',\n                           TEST_IMG_DIR, test_raw_records)\n        gc.collect()\n        GPU.free()\n\n    # Vectorize — dùng vocabulary từ Phase 1\n    test_ids_vec, X_test_sparse = OpcodeOps.vectorize(\n        test_raw_records, top_uni, top_bi\n    )\n    df_test_tab = pd.DataFrame(\n        X_test_sparse.toarray(),\n        index=test_ids_vec,\n        columns=top_uni + top_bi\n    )\n    df_test_tab.index.name = 'Id'\n    df_test_tab.to_csv(TEST_TAB_PATH)\n\n    # Metadata\n    test_bytes_sizes, test_asm_sizes = get_file_sizes_by_ext(COMP_DIR / 'test.7z')\n    test_meta_rows = []\n    for sid in TEST_IDS:\n        bs  = test_bytes_sizes.get(sid, 0)\n        as_ = test_asm_sizes.get(sid, 0)\n        test_meta_rows.append({\n            'Id': sid, 'bytes_size': bs,\n            'asm_size': as_, 'size_ratio': as_/bs if bs > 0 else 0.0,\n        })\n    df_test_meta = pd.DataFrame(test_meta_rows).set_index('Id')\n    df_test_meta['unique_opcode_count'] = (df_test_tab > 0).sum(axis=1)\n    df_test_meta.to_csv(TEST_META_PATH)\n\n    # CNN\n    model = CNNExtractor().to(DEVICE).eval()\n    test_dataset = MalwareImageDataset(TEST_IDS, TEST_IMG_DIR)\n    test_loader  = DataLoader(test_dataset, batch_size=64, shuffle=False,\n                              num_workers=4, pin_memory=True)\n    test_feats, test_ids_cnn = [], []\n    with torch.no_grad():\n        for imgs, ids in tqdm(test_loader, desc='CNN test', ncols=80):\n            test_feats.append(model(imgs.to(DEVICE)).cpu().numpy())\n            test_ids_cnn.extend(ids)\n    cnn_feat_test = np.vstack(test_feats)\n    np.save(TEST_CNN_PATH, cnn_feat_test)\n\n    # FIX 3: Sanity check\n    assert cnn_feat_test.std() > 0.1, f\"CNN zeros! std={cnn_feat_test.std():.4f}\"\n    log.info('CNN sanity: mean=%.3f std=%.3f zeros=%.1f%%',\n             cnn_feat_test.mean(), cnn_feat_test.std(),\n             (cnn_feat_test == 0).mean() * 100)\n\n    del model\n    torch.cuda.empty_cache()\n    log.info('Test done: CNN=%s', cnn_feat_test.shape)\n\ncnn_feat_test = np.load(TEST_CNN_PATH)\ndf_test_tab   = pd.read_csv(TEST_TAB_PATH, index_col='Id')\ndf_test_meta  = pd.read_csv(TEST_META_PATH, index_col='Id')\nlog.info('Test loaded: CNN=%s | tab=%s', cnn_feat_test.shape, df_test_tab.shape)","metadata":{"execution":{"iopub.execute_input":"2026-04-08T14:21:40.436199Z","iopub.status.busy":"2026-04-08T14:21:40.435919Z","iopub.status.idle":"2026-04-08T16:46:26.372776Z","shell.execute_reply":"2026-04-08T16:46:26.372048Z","shell.execute_reply.started":"2026-04-08T14:21:40.436177Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Assemble full feature matrix","metadata":{}},{"cell_type":"code","source":"# ── Align tất cả features theo thứ tự ALL_IDS ─────────────────────────────────\n# CNN features — reorder theo ALL_IDS\ncnn_id_order = {sid: i for i, sid in enumerate(cnn_ids_train)}\ncnn_idx      = [cnn_id_order[sid] for sid in ALL_IDS]\nX_cnn_train  = cnn_feat_train[cnn_idx]           # (N, 1280)\n\n# Tabular features — reindex\nX_tab_train  = df_tab.reindex(ALL_IDS).fillna(0).values    # (N, 525)\n\n# Metadata features — reindex\ndf_meta['unique_opcode_count'] = (df_tab > 0).sum(axis=1)\nX_meta_train = df_meta.reindex(ALL_IDS).fillna(0).values   # (N, 4) ✅\nX_test_meta  = df_test_meta.reindex(TEST_IDS).fillna(0).values  # (M, 4) ✅\n\n\n# Labels\ny_train = df_labels.set_index('Id').reindex(ALL_IDS)['label'].values  # 0-indexed\n\n# Concat tất cả\nX_train = np.hstack([X_cnn_train, X_tab_train, X_meta_train])\nlog.info('X_train shape: %s (CNN=%d + Tab=%d + Meta=%d)',\n         X_train.shape, X_cnn_train.shape[1],\n         X_tab_train.shape[1], X_meta_train.shape[1])\n\n# Test features\nX_test_cnn  = cnn_feat_test                                              # (M, 1280)\nX_test_tab  = df_test_tab.reindex(TEST_IDS).fillna(0).values            # (M, 525)\nX_test = np.hstack([X_test_cnn, X_test_tab, X_test_meta])\nlog.info('X_test shape : %s', X_test.shape)\n\nprint(f'\\nX_train : {X_train.shape}  (total {X_train.shape[1]} features)')\nprint(f'X_test  : {X_test.shape}')\nprint(f'y_train : {y_train.shape}  classes={np.unique(y_train)}')\nprint(f'\\nClass distribution:')\nfor c, n in sorted(zip(*np.unique(y_train, return_counts=True))):\n    print(f'  Class {c+1} ({CLASS_NAMES[c+1]:<18}): {n:4d} ({n/len(y_train)*100:.1f}%)')","metadata":{"execution":{"iopub.execute_input":"2026-04-08T16:46:26.374871Z","iopub.status.busy":"2026-04-08T16:46:26.374560Z","iopub.status.idle":"2026-04-08T16:46:27.035850Z","shell.execute_reply":"2026-04-08T16:46:27.035223Z","shell.execute_reply.started":"2026-04-08T16:46:26.374787Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Feature Selection (Random Forest)","metadata":{}},{"cell_type":"code","source":"TOP_K = 1000   # giữ top-1000 features\nSEL_PATH = FEAT_DIR / 'selected_indices.npy'\n\nif SEL_PATH.exists():\n    selected_idx = np.load(SEL_PATH)\n    log.info('Feature indices loaded: %d features', len(selected_idx))\nelse:\n    log.info('Fitting Random Forest for feature importance (n_estimators=200)...')\n    t0 = time.time()\n\n    rf = RandomForestClassifier(\n        n_estimators=200,\n        max_depth=15,\n        n_jobs=-1,\n        random_state=SEED\n    )\n    rf.fit(X_train, y_train)\n\n    importances  = rf.feature_importances_\n    selected_idx = np.argsort(importances)[::-1][:TOP_K]\n    np.save(SEL_PATH, selected_idx)\n\n    log.info('Feature selection done in %.1f min', (time.time()-t0)/60)\n    log.info('Top 10 feature indices: %s', selected_idx[:10])\n\n# Apply selection\nX_train_sel = X_train[:, selected_idx]   # (N, 1000)\nX_test_sel  = X_test[:, selected_idx]    # (M, 1000)\n\nlog.info('After selection: train=%s  test=%s', X_train_sel.shape, X_test_sel.shape)","metadata":{"execution":{"iopub.execute_input":"2026-04-08T16:46:27.037390Z","iopub.status.busy":"2026-04-08T16:46:27.037058Z","iopub.status.idle":"2026-04-08T16:46:49.964626Z","shell.execute_reply":"2026-04-08T16:46:49.964004Z","shell.execute_reply.started":"2026-04-08T16:46:27.037366Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Stratified K-Fold XGBoost Training","metadata":{}},{"cell_type":"code","source":"# ── XGBoost params ────────────────────────────────────────────────────────────\nXGB_PARAMS = {\n    'objective'        : 'multi:softprob',\n    'num_class'        : N_CLASSES,\n    'eval_metric'      : 'mlogloss',\n    'max_depth'        : 7,\n    'learning_rate'    : 0.05,\n    'subsample'        : 0.8,\n    'colsample_bytree' : 0.8,\n    'min_child_weight' : 3,\n    'gamma'            : 0.1,\n    'tree_method'      : 'hist',   # GPU acceleration\n    'device'           : 'cuda',\n    'seed'             : SEED,\n    'verbosity'        : 0,\n}\n\n# ── sample_weight cho class imbalance ─────────────────────────────────────────\n# Đúng cách cho multiclass — scale_pos_weight bị ignore trong multi:softprob\nsample_weights = compute_sample_weight('balanced', y_train)\nlog.info('Sample weight range: [%.3f, %.3f]',\n         sample_weights.min(), sample_weights.max())\n\n# ── Stratified K-Fold ─────────────────────────────────────────────────────────\nskf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\noof_preds   = np.zeros((len(y_train), N_CLASSES))  # Out-of-fold predictions\ntest_preds  = np.zeros((len(TEST_IDS), N_CLASSES)) # Test predictions\nfold_models = []\nfold_scores = []\n\nlog.info('Starting %d-Fold XGBoost training...', N_FOLDS)\nt_total = time.time()\n\nfor fold, (tr_idx, val_idx) in enumerate(\n        skf.split(X_train_sel, y_train)):\n\n    t_fold = time.time()\n    log.info('Fold %d/%d ...', fold+1, N_FOLDS)\n\n    X_tr, X_val = X_train_sel[tr_idx], X_train_sel[val_idx]\n    y_tr, y_val = y_train[tr_idx],     y_train[val_idx]\n    w_tr        = sample_weights[tr_idx]\n\n    dtrain = xgb.DMatrix(X_tr,  label=y_tr,  weight=w_tr)\n    dval   = xgb.DMatrix(X_val, label=y_val)\n    dtest  = xgb.DMatrix(X_test_sel)\n\n    model = xgb.train(\n        XGB_PARAMS,\n        dtrain,\n        num_boost_round=1000,\n        evals=[(dval, 'val')],\n        early_stopping_rounds=30,\n        verbose_eval=100\n    )\n\n    # OOF predictions\n    val_prob = model.predict(dval)                # (val_size, 9)\n    oof_preds[val_idx] = val_prob\n\n    # Test predictions (accumulate)\n    test_preds += model.predict(dtest) / N_FOLDS\n\n    # Score\n    val_acc  = accuracy_score(y_val, val_prob.argmax(axis=1))\n    val_loss = log_loss(y_val, val_prob)\n    fold_scores.append({'acc': val_acc, 'loss': val_loss})\n    fold_models.append(model)\n\n    log.info('Fold %d | acc=%.4f | log_loss=%.4f | %.1fs',\n             fold+1, val_acc, val_loss, time.time()-t_fold)\n\n# ── OOF overall score ─────────────────────────────────────────────────────────\noof_acc  = accuracy_score(y_train, oof_preds.argmax(axis=1))\noof_loss = log_loss(y_train, oof_preds)\nlog.info('OOF overall | acc=%.4f | log_loss=%.4f | total=%.1f min',\n         oof_acc, oof_loss, (time.time()-t_total)/60)\n\nprint(f'\\n{\"─\"*40}')\nprint(f' OOF Accuracy : {oof_acc:.4f}')\nprint(f' OOF Log-loss : {oof_loss:.4f}')\nprint(f'{'─'*40}')\nfor i, s in enumerate(fold_scores):\n    print(f' Fold {i+1}: acc={s[\"acc\"]:.4f}  loss={s[\"loss\"]:.4f}')","metadata":{"execution":{"iopub.execute_input":"2026-04-08T16:46:49.966206Z","iopub.status.busy":"2026-04-08T16:46:49.965880Z","iopub.status.idle":"2026-04-08T16:48:14.681190Z","shell.execute_reply":"2026-04-08T16:48:14.680510Z","shell.execute_reply.started":"2026-04-08T16:46:49.966180Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM Training \n","metadata":{}},{"cell_type":"code","source":"LGB_PARAMS = {\n    'objective'       : 'multiclass',\n    'num_class'       : N_CLASSES,\n    'metric'          : 'multi_logloss',\n    'n_estimators'    : 2000,\n    'learning_rate'   : 0.05,\n    'max_depth'       : 7,\n    'num_leaves'      : 63,         \n    'subsample'       : 0.8,\n    'colsample_bytree': 0.8,\n    'min_child_samples': 10,\n    'reg_alpha'       : 0.1,\n    'reg_lambda'      : 1.0,\n    'device'          : 'gpu', \n    'seed'            : SEED,\n    'verbose'         : -1,\n}\n\nskf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\nlgb_oof   = np.zeros((len(y_train), N_CLASSES))\nlgb_test  = np.zeros((X_test_sel.shape[0], N_CLASSES))\nlgb_models= []\nlgb_scores= []\n\nprint('LightGBM 5-Fold training...')\nt0 = time.time()\n\nfor fold, (tr_idx, val_idx) in enumerate(skf.split(X_train_sel, y_train)):\n    X_tr, X_val = X_train_sel[tr_idx], X_train_sel[val_idx]\n    y_tr, y_val = y_train[tr_idx],     y_train[val_idx]\n    w_tr        = sample_weights[tr_idx]\n\n    model = lgb.LGBMClassifier(**LGB_PARAMS)\n    model.fit(\n        X_tr, y_tr,\n        sample_weight=w_tr,\n        eval_set=[(X_val, y_val)],\n        callbacks=[\n            lgb.early_stopping(30, verbose=False),\n            lgb.log_evaluation(100)\n        ]\n    )\n\n    val_prob = model.predict_proba(X_val)\n    lgb_oof[val_idx] = val_prob\n    lgb_test += model.predict_proba(X_test_sel) / N_FOLDS\n\n    val_acc  = accuracy_score(y_val, val_prob.argmax(axis=1))\n    val_loss = log_loss(y_val, val_prob)\n    lgb_scores.append({'acc': val_acc, 'loss': val_loss})\n    lgb_models.append(model)\n    print(f'  Fold {fold+1} | acc={val_acc:.4f} | loss={val_loss:.4f}')\n\nlgb_oof_acc  = accuracy_score(y_train, lgb_oof.argmax(axis=1))\nlgb_oof_loss = log_loss(y_train, lgb_oof)\nprint(f'\\nLightGBM OOF | acc={lgb_oof_acc:.4f} | loss={lgb_oof_loss:.4f}')\nprint(f'XGBoost OOF  | acc={accuracy_score(y_train, oof_preds.argmax(1)):.4f} | loss={log_loss(y_train, oof_preds):.4f}')\nprint(f'Time: {(time.time()-t0)/60:.1f} min')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Ensemble XGB + LLGB qua weight average ","metadata":{}},{"cell_type":"code","source":"from scipy.optimize import minimize_scalar\n\nEPS = 1e-5\n\ndef clip_norm(probs, eps=EPS):\n    probs = np.clip(probs, eps, 1 - eps)\n    return probs / probs.sum(axis=1, keepdims=True)\n\n# Clip & normalize OOF và test predictions\nxgb_oof_clipped  = clip_norm(oof_preds.copy())\nlgb_oof_clipped  = clip_norm(lgb_oof.copy())\nxgb_test_clipped = clip_norm(test_preds.copy())\nlgb_test_clipped = clip_norm(lgb_test.copy())\n\n# Tìm alpha tối ưu trên OOF: minimize log_loss(alpha*XGB + (1-alpha)*LGB)\ndef ensemble_oof_loss(alpha):\n    combined = alpha * xgb_oof_clipped + (1 - alpha) * lgb_oof_clipped\n    return log_loss(y_train, np.clip(combined, EPS, 1 - EPS))\n\nresult     = minimize_scalar(ensemble_oof_loss, bounds=(0, 1), method='bounded')\nbest_alpha = result.x\nbest_loss  = result.fun\n\n# OOF ensemble\nens_oof  = clip_norm(best_alpha * xgb_oof_clipped + (1 - best_alpha) * lgb_oof_clipped)\n# Test ensemble — dùng làm teacher cho pseudo-labeling\nens_test = clip_norm(best_alpha * xgb_test_clipped + (1 - best_alpha) * lgb_test_clipped)\n\nens_oof_acc  = accuracy_score(y_train, ens_oof.argmax(axis=1))\nens_oof_loss = log_loss(y_train, ens_oof)\n\nprint(f'Optimal alpha (XGB weight) : {best_alpha:.3f}')\nprint(f'  → XGB weight={best_alpha:.3f}  LGB weight={1-best_alpha:.3f}')\nprint()\nprint('OOF log-loss comparison:')\nprint(f'  XGB only       : {log_loss(y_train, xgb_oof_clipped):.4f}')\nprint(f'  LGB only       : {log_loss(y_train, lgb_oof_clipped):.4f}')\nprint(f'  Weighted Avg   : {ens_oof_loss:.4f}  ← best')\nprint()\nprint(f'Ensemble OOF acc : {ens_oof_acc:.4f}')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"##  Pseudo-labeling ","metadata":{}},{"cell_type":"code","source":"# Pseudo-labeling với Weighted Ensemble làm teacher\n# Teacher = ens_test (XGB+LGB ensemble) =\n\nPSEUDO_THRESHOLD = 0.98\nMAX_PSEUDO_ROUNDS = 2\n\n# Teacher: ensemble predictions (không phải single XGB — tránh self-labeling bias)\nteacher_test_probs = ens_test.copy()\nlog.info('Pseudo teacher: weighted_ensemble (alpha=%.3f)', best_alpha)\n\n# Baseline từ OOF thật (5-fold holdout), không phải in-sample\nbest_oof_probs = ens_oof.copy()\nbest_oof_acc   = ens_oof_acc\nbest_oof_loss  = ens_oof_loss\n\n# XGB test predictions — sẽ được update qua mỗi pseudo round\ncurrent_xgb_test = xgb_test_clipped.copy()\n\n# Pseudo pool\nX_pseudo_pool  = np.empty((0, X_train_sel.shape[1]), dtype=X_train_sel.dtype)\ny_pseudo_pool  = np.empty((0,), dtype=y_train.dtype)\nused_test_mask = np.zeros(len(TEST_IDS), dtype=bool)\n\n# Best rounds từ fold models\nbest_round_candidates = [getattr(m, 'best_iteration', None) for m in fold_models]\nbest_round_candidates = [r for r in best_round_candidates if r is not None and r > 0]\nbest_rounds = int(np.mean(best_round_candidates)) if best_round_candidates else 300\nlog.info('Pseudo num_boost_round: %d', best_rounds)\n\npseudo_skf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\nfor pseudo_round in range(MAX_PSEUDO_ROUNDS):\n    max_probs    = teacher_test_probs.max(axis=1)\n    pseudo_y_all = teacher_test_probs.argmax(axis=1)\n\n    new_mask = (max_probs >= PSEUDO_THRESHOLD) & (~used_test_mask)\n    n_new    = int(new_mask.sum())\n    log.info('Pseudo round %d: %d/%d samples (threshold=%.2f)',\n             pseudo_round + 1, n_new, len(TEST_IDS), PSEUDO_THRESHOLD)\n\n    if n_new == 0:\n        log.info('No new pseudo labels — stop.')\n        break\n\n    # Candidate pool\n    X_pseudo_candidate = np.vstack([X_pseudo_pool, X_test_sel[new_mask]])\n    y_pseudo_candidate = np.concatenate([y_pseudo_pool, pseudo_y_all[new_mask]])\n\n    # Proper OOF: mỗi fold train KHÔNG thấy validation labels\n    pseudo_oof = np.zeros((len(y_train), N_CLASSES))\n    for fold, (tr_idx, val_idx) in enumerate(\n            pseudo_skf.split(X_train_sel, y_train), start=1):\n\n        X_fold_tr = np.vstack([X_train_sel[tr_idx], X_pseudo_candidate])\n        y_fold_tr = np.concatenate([y_train[tr_idx], y_pseudo_candidate])\n        w_fold_tr = compute_sample_weight('balanced', y_fold_tr)\n\n        dtr  = xgb.DMatrix(X_fold_tr, label=y_fold_tr, weight=w_fold_tr)\n        dval = xgb.DMatrix(X_train_sel[val_idx], label=y_train[val_idx])\n\n        fold_model = xgb.train(\n            {**XGB_PARAMS, 'verbosity': 0},\n            dtr,\n            num_boost_round=best_rounds,\n            evals=[(dval, 'val')],\n            verbose_eval=False\n        )\n        pseudo_oof[val_idx] = fold_model.predict(dval)\n\n    pseudo_oof     = clip_norm(pseudo_oof)\n    new_oof_acc    = accuracy_score(y_train, pseudo_oof.argmax(axis=1))\n    new_oof_loss   = log_loss(y_train, pseudo_oof)\n\n    log.info('Round %d OOF | acc=%.4f (was %.4f) | loss=%.4f (was %.4f)',\n             pseudo_round + 1, new_oof_acc, best_oof_acc,\n             new_oof_loss, best_oof_loss)\n\n    if new_oof_loss >= best_oof_loss - 1e-4:\n        log.info('Not improved on proper OOF — stop.')\n        break\n\n    # Accept round\n    X_pseudo_pool  = X_pseudo_candidate\n    y_pseudo_pool  = y_pseudo_candidate\n    used_test_mask[new_mask] = True\n\n    # Retrain XGB trên toàn bộ labeled + pseudo\n    X_aug = np.vstack([X_train_sel, X_pseudo_pool])\n    y_aug = np.concatenate([y_train, y_pseudo_pool])\n    w_aug = compute_sample_weight('balanced', y_aug)\n\n    pseudo_model    = xgb.train(\n        {**XGB_PARAMS, 'verbosity': 0},\n        xgb.DMatrix(X_aug, label=y_aug, weight=w_aug),\n        num_boost_round=best_rounds\n    )\n    current_xgb_test = clip_norm(pseudo_model.predict(xgb.DMatrix(X_test_sel)))\n\n    best_oof_probs = pseudo_oof\n    best_oof_acc   = new_oof_acc\n    best_oof_loss  = new_oof_loss\n    log.info('✅ Pseudo round %d accepted. Pool size: %d', pseudo_round+1, len(y_pseudo_pool))\n\n# Export: XGB test đã được update qua pseudo\nxgb_test_final = current_xgb_test\nlog.info('Pseudo done. Final XGB test shape: %s', xgb_test_final.shape)\n","metadata":{"execution":{"iopub.execute_input":"2026-04-08T16:48:14.682534Z","iopub.status.busy":"2026-04-08T16:48:14.682257Z","iopub.status.idle":"2026-04-08T16:49:11.833206Z","shell.execute_reply":"2026-04-08T16:49:11.832598Z","shell.execute_reply.started":"2026-04-08T16:48:14.682513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Final Ensemble: recompute alpha với XGB post-pseudo OOF + LGB OOF\n# best_oof_probs là XGB OOF sau pseudo (proper 5-fold), lgb_oof_clipped không đổi\n\ndef clip_norm(probs, eps=EPS):\n    probs = np.clip(probs, eps, 1 - eps)\n    return probs / probs.sum(axis=1, keepdims=True)\n\nxgb_final_oof = clip_norm(best_oof_probs)   # XGB post-pseudo OOF\n\ndef final_ensemble_loss(alpha):\n    combined = alpha * xgb_final_oof + (1 - alpha) * lgb_oof_clipped\n    return log_loss(y_train, np.clip(combined, EPS, 1 - EPS))\n\nresult_final     = minimize_scalar(final_ensemble_loss, bounds=(0, 1), method='bounded')\nfinal_alpha      = result_final.x\nfinal_oof_combo  = clip_norm(final_alpha * xgb_final_oof + (1 - final_alpha) * lgb_oof_clipped)\nfinal_test_combo = clip_norm(final_alpha * xgb_test_final + (1 - final_alpha) * lgb_test_clipped)\n\nfinal_oof_acc  = accuracy_score(y_train, final_oof_combo.argmax(axis=1))\nfinal_oof_loss = log_loss(y_train, final_oof_combo)\n\nprint('=== Final Ensemble Comparison ===')\nprint(f'  XGB post-pseudo OOF  : {log_loss(y_train, xgb_final_oof):.4f}')\nprint(f'  LGB OOF              : {log_loss(y_train, lgb_oof_clipped):.4f}')\nprint(f'  Pre-pseudo ensemble  : {ens_oof_loss:.4f}  (alpha={best_alpha:.3f})')\nprint(f'  Final ensemble       : {final_oof_loss:.4f}  (alpha={final_alpha:.3f})  ← submit')\nprint()\n\n# ── OOF Analysis ──────────────────────────────────────────────────────────────\nfrom sklearn.metrics import classification_report\nprint('OOF CONFUSION MATRIX (final ensemble):')\nprint(classification_report(\n    y_train, final_oof_combo.argmax(axis=1),\n    target_names=[CLASS_NAMES[i+1] for i in range(9)]\n))\n\nprint('Per-class accuracy:')\nfor c in range(9):\n    mask = y_train == c\n    if mask.sum() > 0:\n        acc  = (final_oof_combo.argmax(axis=1)[mask] == c).mean()\n        flag = '⚠' if acc < 0.95 else '✅'\n        print(f'  {flag} Class {c+1} ({CLASS_NAMES[c+1]:<18}): {acc:.4f}  (n={mask.sum()})')\n\ndf_sample = pd.read_csv(COMP_DIR / 'sampleSubmission.csv')\nprob_cols  = [c for c in df_sample.columns if c != 'Id'] \n\ndf_submission = pd.DataFrame(final_test_combo, columns=prob_cols)\ndf_submission.insert(0, 'Id', TEST_IDS)\n\nassert df_submission.shape == (len(TEST_IDS), 10), f'Wrong shape: {df_submission.shape}'\nassert np.allclose(df_submission[prob_cols].sum(axis=1), 1.0, atol=1e-4), 'Probs not sum to 1'\n\nsub_path = WORK_DIR / 'submission.csv'\ndf_submission.to_csv(sub_path, index=False)\n\nprint(f'\\nSubmission saved: {sub_path}')\nprint(f'Shape: {df_submission.shape}  ✅')\nprint('\\nClass distribution (argmax):')\nfinal_class = final_test_combo.argmax(axis=1) + 1\nprint(pd.Series(final_class).value_counts().sort_index().to_string())\n\nprint(f'\\n{\"═\"*45}')\nprint('  FINAL RESULTS')\nprint(f'  OOF Accuracy  : {final_oof_acc:.4f}')\nprint(f'  OOF Log-loss  : {final_oof_loss:.4f}')\nprint(f'  XGB alpha     : {final_alpha:.3f}  LGB alpha: {1-final_alpha:.3f}')\nprint('  Target        : ≥ 0.99')\nstatus = '✅ ĐẠT' if final_oof_acc >= 0.99 else '⚠  CHƯA ĐẠT'\nprint(f'  Status        : {status}')\nprint(f'{\"═\"*45}')\n","metadata":{"execution":{"iopub.status.busy":"2026-04-16T09:29:30.086890Z","iopub.execute_input":"2026-04-16T09:29:30.087471Z","iopub.status.idle":"2026-04-16T09:29:30.109412Z","shell.execute_reply.started":"2026-04-16T09:29:30.087443Z","shell.execute_reply":"2026-04-16T09:29:30.108057Z"},"trusted":true},"outputs":[],"execution_count":null}]}