{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665},{"sourceType":"datasetVersion","sourceId":15913569,"datasetId":10205228,"databundleVersionId":16869599}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phase 1 — Full Feature Extraction (V7 Ultimate)\n\n> Skip if precomputed dataset available.\n","metadata":{}},{"cell_type":"markdown","source":"## A. Bootstrap\n","metadata":{}},{"cell_type":"code","source":"import sys, gc, time, logging, zipfile, shutil\nimport numpy as np, pandas as pd\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\n\nCOMP_DIR   = Path(\"/kaggle/input/competitions/malware-classification\")\nUTILS_PATH = Path(\"/kaggle/input/datasets/bnthanh/malware-utils-v7\")\nif str(UTILS_PATH) not in sys.path: sys.path.append(str(UTILS_PATH))\nfrom malware_utils_v7 import (Config, GPU, FileOps, ImageOps, OpcodeOps,\n                               CacheOps, ByteEntropyOps, MarkovV2Ops, AsmStructOps)\n\nlogging.basicConfig(level=logging.INFO,\n    format=\"%(asctime)s [%(levelname)s] %(message)s\", datefmt=\"%H:%M:%S\",\n    handlers=[logging.StreamHandler(sys.stdout),\n              logging.FileHandler(\"/kaggle/working/p1.log\")])\nlog = logging.getLogger(\"p1\")\n\ncfg = Config(base_dir=COMP_DIR, batch_n=80, unigram_k=500, bigram_k=1000, trigram_k=500, min_free_gb=5.0)\ncfg.make_dirs(); GPU.setup()\nlog.info(\"Disk=%.1f GB  train.7z=%s\", FileOps.get_free_gb(), cfg.train_7z.exists())\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## B. Labels\n","metadata":{}},{"cell_type":"code","source":"df_labels = pd.read_csv(cfg.label_file); df_labels.columns = df_labels.columns.str.strip()\nfor col in df_labels.columns:\n    if col.lower() in (\"id\",\"name\"): df_labels = df_labels.rename(columns={col:\"Id\"})\n    if col.lower() == \"class\":       df_labels = df_labels.rename(columns={col:\"Class\"})\nALL_IDS = df_labels[\"Id\"].tolist()\ncached = CacheOps.get_cached_ids(cfg.cache_img_dir)\nexisting = {r[\"id\"] for r in CacheOps.load_raw_counts(cfg.raw_counts_file)}\ntodo = [s for s in ALL_IDS if s not in cached and s not in existing]\nif todo:\n    todo = FileOps.sort_by_archive_order(todo, FileOps.get_archive_order(cfg.train_7z))\nbatches = list(FileOps.chunked(todo, cfg.batch_n))\nlog.info(\"Total=%d | Done=%d | Todo=%d | Batches=%d\", len(ALL_IDS), len(ALL_IDS)-len(todo), len(todo), len(batches))\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## C. Batch Extraction — V7: All feature streams\n","metadata":{}},{"cell_type":"code","source":"import pickle\nt0 = time.time()\nCacheOps.clear_batch_records(cfg.batch_records_dir)\n\ndef extract_v7(sid, bp, ap):\n    \"\"\"V7: asm+bytes+entropy+markov2+struct in one call.\"\"\"\n    c = OpcodeOps.parse_advanced(asm_path=ap, bytes_path=bp)\n    c[\"id\"] = sid\n    bs  = bp.stat().st_size if bp else 0\n    as_ = ap.stat().st_size if ap else 0\n    c[\"meta\"] = {\"bytes_size\":bs,\"asm_size\":as_,\"size_ratio\":as_/bs if bs>0 else 0.}\n    # Byte entropy (535 features: 256 freq + 20 ent_hist + 256 cond_ent + 3 scalars)\n    c[\"byte_entropy\"] = ByteEntropyOps.extract_all(bp) if bp else {}\n    # Markov order-2 (256 cond-ent features)\n    c[\"markov2\"]      = MarkovV2Ops.extract(bp)        if bp else {}\n    # ASM structural (9 features)\n    c[\"asm_struct\"]   = AsmStructOps.extract(ap)       if ap else {}\n    return c\n\nfor bi, batch_ids in enumerate(batches):\n    if FileOps.get_free_gb() < cfg.min_free_gb: break\n    log.info(\"Batch %03d/%03d\", bi+1, len(batches))\n    pairs = FileOps.extract_batch(cfg.train_7z, batch_ids, cfg.scratch_dir)\n    recs  = []\n    for sid, bp, ap in pairs:\n        try:\n            CacheOps.save_image(ImageOps.process_image(bp,ap,cfg), sid, cfg.cache_img_dir)\n            recs.append(extract_v7(sid, bp, ap))\n        except Exception as e: log.debug(\"Fail %s: %s\", sid, e)\n        FileOps.delete_raw(bp, ap)\n    CacheOps.save_batch_records(recs, bi, cfg.batch_records_dir)\n    gc.collect(); GPU.free()\n    shutil.rmtree(cfg.scratch_dir, ignore_errors=True); cfg.scratch_dir.mkdir(exist_ok=True)\nlog.info(\"Pass1 done: %.1f min\", (time.time()-t0)/60)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## D. Vocabulary + Tabular\n","metadata":{}},{"cell_type":"code","source":"vocab = OpcodeOps.build_vocabulary(\n    CacheOps.iter_all_records(cfg.batch_records_dir),\n    unigram_k=500, bigram_k=1000, trigram_k=500,\n    section_k=300, api_k=200, byte_uni_k=256, byte_bi_k=512, dll_k=200)\nwith open(cfg.features_dir/\"vocab_full.pkl\",\"wb\") as f: pickle.dump(vocab,f,protocol=4)\nlog.info(\"vocab: %s\", {k:len(v) for k,v in vocab.items()})\n\nSTAT_COLS = [\"n_funcs\",\"n_opcodes\",\"unique_ops\"]\n\ndf_tab = OpcodeOps.create_feature_df(\n    CacheOps.iter_all_records(cfg.batch_records_dir), vocab, STAT_COLS)\n\n# Merge all extra feature blocks\nextra_blocks = {\"byte_entropy\":[], \"markov2\":[], \"asm_struct\":[]}\nextra_ids    = []\nfor r in CacheOps.iter_all_records(cfg.batch_records_dir):\n    extra_ids.append(r[\"id\"])\n    for k in extra_blocks: extra_blocks[k].append(r.get(k, {}))\n\nfor block_name, rows in extra_blocks.items():\n    df_extra = pd.DataFrame(rows, index=extra_ids, dtype=np.float32)\n    df_extra.index.name = \"Id\"\n    df_tab = df_tab.join(df_extra, how=\"left\").fillna(0)\n\ndf_tab = df_tab.merge(df_labels[[\"Id\",\"Class\"]].set_index(\"Id\"),\n                      left_index=True, right_index=True, how=\"left\")\ndf_tab.to_csv(cfg.features_dir/\"tabular_features.csv\")\nlog.info(\"tabular_features.csv: %s cols (target >6000 for v7)\", df_tab.shape)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## E. Package\n","metadata":{}},{"cell_type":"code","source":"import zipfile\nnpy_files = list(cfg.cache_img_dir.glob(\"*.npy\"))\nprint(f\"Images: {len(npy_files)}/{len(ALL_IDS)}  Disk: {FileOps.get_free_gb():.1f}GB\")\nzp = cfg.features_dir/\"image_features.zip\"\nwith zipfile.ZipFile(zp,\"w\",compression=zipfile.ZIP_STORED,allowZip64=True) as zf:\n    for p in tqdm(npy_files,desc=\"Zip\"): zf.write(p,arcname=p.name)\nwith zipfile.ZipFile(zp) as zf: nz=len(zf.namelist())\nif nz==len(npy_files): shutil.rmtree(cfg.cache_img_dir,ignore_errors=True)\ncfg.raw_counts_file.unlink(missing_ok=True)\nlog.info(\"Phase 1 complete: %d images, %d feature cols\", nz, df_tab.shape[1])\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.exit(\"Đã hoàn thành Phase 1. Dừng Notebook chờ chạy Phase 2!\")","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Phase 2 — V7 ULTIMATE Pipeline\n\n## Key changes vs V6\n| Component | V6 | V7 |\n|-----------|-------|-------|\n| Tabular features | ~5800 | **~6600** (+markov2 +asm_struct) |\n| CNNs | EffB3 only | **EffB3 + ConvNeXt-Small** (dual-stream) |\n| CNN augment | 16-fold TTA | **16-fold TTA + MixUp** |\n| GBDT models | XGB+LGB-DART+CB+ET | **same + XGB as L2 meta** |\n| L2 meta-learner | MLP (sklearn) | **XGBoost meta** (more powerful) |\n| Calibration | Temperature | **Per-class isotonic regression** |\n| Pseudo-labeling | 0.998 thresh | **0.999 + 3 rounds** |\n| Feature selection | top-2000 RF | **top-2500 + permutation importance** |\n","metadata":{}},{"cell_type":"markdown","source":"## A. Setup\n","metadata":{}},{"cell_type":"code","source":"import sys, os, gc, time, logging, zipfile, shutil, pickle\nimport numpy as np, pandas as pd\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom scipy.optimize import minimize\n\nimport torch, torch.nn as nn, torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.models as models\nimport torchvision.transforms as T\n\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier, Pool\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\nfrom sklearn.isotonic import IsotonicRegression\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.utils.class_weight import compute_sample_weight\nfrom sklearn.metrics import accuracy_score, log_loss\n\nfor h in logging.root.handlers[:]: logging.root.removeHandler(h)\nlogging.basicConfig(level=logging.INFO,\n    format=\"%(asctime)s [%(levelname)s] %(message)s\", datefmt=\"%H:%M:%S\",\n    handlers=[logging.StreamHandler(sys.stdout),\n              logging.FileHandler(\"/kaggle/working/v7.log\")])\nlog = logging.getLogger(\"v7\")\n\nSEED, N_FOLDS, N_CLASSES, CNN_FOLDS = 42, 10, 9, 5\nCLASS_NAMES = {1:\"Ramnit\",2:\"Lollipop\",3:\"Kelihos_ver3\",4:\"Vundo\",5:\"Simda\",\n               6:\"Tracur\",7:\"Kelihos_ver1\",8:\"Obfuscator.ACY\",9:\"Gatak\"}\n\nWORK_DIR = Path(\"/kaggle/working\")\nCOMP_DIR = Path(\"/kaggle/input/competitions/malware-classification\")\nIMG_DIR  = WORK_DIR/\"train_images\"\nSEL_PATH = WORK_DIR/\"feature_selection.npy\"\n\nUTILS_PATH = Path(\"/kaggle/input/datasets/bnthanh/malware-utils-v7\")\nif str(UTILS_PATH) not in sys.path: sys.path.append(str(UTILS_PATH))\nfrom malware_utils_v7 import (Config, GPU, FileOps, ImageOps, OpcodeOps,\n                               CacheOps, ByteEntropyOps, MarkovV2Ops, AsmStructOps)\ncfg = Config(base_dir=COMP_DIR); GPU.setup()\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nDL_TR = dict(batch_size=16, shuffle=True,  num_workers=0, pin_memory=False)\nDL_EX = dict(batch_size=8,  shuffle=False, num_workers=0, pin_memory=False)\n\n_PRE = Path(\"/kaggle/input/datasets/bnthanh/malwareclassificationpreprocessdata/features\")\nHAS_PRE = _PRE.exists() and (_PRE/\"cnn_oof_effb3.npy\").exists() and (_PRE/\"tabular_features.csv\").exists()\nHAS_PRE = False # để chạy từ đầu\nif HAS_PRE:\n    FEAT_DIR=_PRE; MODE=\"FAST\"\nelse:\n    FEAT_DIR=WORK_DIR/\"features\"; FEAT_DIR.mkdir(parents=True,exist_ok=True); MODE=\"FULL\"\nIMG_DIR.mkdir(parents=True,exist_ok=True)\n(WORK_DIR/\"features\").mkdir(parents=True,exist_ok=True)\nlog.info(\"Mode=%s Device=%s Disk=%.1fGB\", MODE, DEVICE, FileOps.get_free_gb())\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## B. Load Features\n","metadata":{}},{"cell_type":"code","source":"df_labels = pd.read_csv(COMP_DIR/\"trainLabels.csv\")\ndf_labels.columns = df_labels.columns.str.strip()\nfor c in df_labels.columns:\n    if c.lower() in (\"id\",\"name\"): df_labels=df_labels.rename(columns={c:\"Id\"})\n    if c.lower()==\"class\":         df_labels=df_labels.rename(columns={c:\"Class\"})\ndf_labels[\"label\"] = df_labels[\"Class\"] - 1\nALL_IDS  = df_labels[\"Id\"].tolist()\nTEST_IDS = pd.read_csv(COMP_DIR/\"sampleSubmission.csv\")[\"Id\"].tolist()\nMETA_COLS = [\"bytes_size\",\"asm_size\",\"size_ratio\"]\nSTAT_COLS = [\"n_funcs\",\"n_opcodes\",\"unique_ops\"]\n\nwith open(FEAT_DIR/\"vocab_full.pkl\",\"rb\") as f: vocab=pickle.load(f)\nlog.info(\"vocab: %s\", {k:len(v) for k,v in vocab.items()})\n\ndf_tab = pd.read_csv(FEAT_DIR/\"tabular_features.csv\", index_col=\"Id\")\nlog.info(\"Tabular: %s\", df_tab.shape)\ndf_meta = df_tab[META_COLS].copy()\ndf_tab  = df_tab.drop(columns=META_COLS)\n_       = df_tab.pop(\"Class\") if \"Class\" in df_tab.columns else None\nprint(f\"Tab: {df_tab.shape}  Meta: {df_meta.shape}  Mode: {MODE}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## C. Images\n","metadata":{}},{"cell_type":"code","source":"# ── C. Extract Train Images (Hybrid Mode - Fix Kaggle Auto-unzip) ───────────────\nimport shutil\nimport zipfile\n\nzip_path       = FEAT_DIR / \"image_features.zip\"\nsource_img_dir = FEAT_DIR / \"image_features\"  # Thư mục Kaggle tự giải nén\nalready        = list(IMG_DIR.glob(\"*.npy\"))\n\n# Đảm bảo thư mục đích tồn tại\nIMG_DIR.mkdir(parents=True, exist_ok=True)\n\nif len(already) >= len(ALL_IDS) > 0:\n    log.info(\"Train images OK: %d files - skip\", len(already))\n    \nelif source_img_dir.exists() and source_img_dir.is_dir():\n    # Kaggle đã tự giải nén lúc Add Data -> Copy sang working dir\n    t0 = time.time()\n    log.info(f\"Phát hiện Folder. Đang copy ảnh từ {source_img_dir} sang {IMG_DIR}...\")\n    shutil.copytree(source_img_dir, IMG_DIR, dirs_exist_ok=True)\n    log.info(\"Đã copy %d file ảnh trong %.1fs\", len(list(IMG_DIR.glob(\"*.npy\"))), time.time()-t0)\n    \nelif zip_path.exists():\n    # Có file Zip -> Giải nén\n    t0 = time.time()\n    log.info(f\"Phát hiện File Zip. Đang giải nén ảnh từ {zip_path}...\")\n    with zipfile.ZipFile(zip_path, \"r\") as zf: \n        zf.extractall(IMG_DIR)\n    log.info(\"Đã giải nén %d file ảnh trong %.1fs\", len(list(IMG_DIR.glob(\"*.npy\"))), time.time()-t0)\n    \nelse:\n    log.error(\"LỖI NGHIÊM TRỌNG: Không tìm thấy thư mục ảnh lẫn file zip trong %s\", FEAT_DIR)\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## D. Dual-CNN: EfficientNet-B3 + ConvNeXt-Small (5-fold OOF each)\n","metadata":{}},{"cell_type":"code","source":"class MalwareDS(Dataset):\n    def __init__(self, ids, img_dir, label_map=None):\n        self.ids,self.img_dir,self.label_map=ids,img_dir,label_map\n        self.tf=T.Compose([T.ToTensor(),T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])])\n    def __len__(self): return len(self.ids)\n    def __getitem__(self,i):\n        s=self.ids[i]; im=self.tf(np.load(self.img_dir/f\"{s}.npy\"))\n        return (im,self.label_map[s]) if self.label_map else (im,s)\n\nclass TTADS(Dataset):\n    N=16\n    def __init__(self,ids,img_dir):\n        self.ids,self.img_dir=ids,img_dir\n        self.norm=T.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])\n    def __len__(self): return len(self.ids)\n    def __getitem__(self,i):\n        s=self.ids[i]\n        t=torch.from_numpy(np.load(self.img_dir/f\"{s}.npy\")).permute(2,0,1).float()/255.\n        augs=[t,t.flip(-1),t.flip(-2),\n              t.rot90(1,[-2,-1]),t.rot90(2,[-2,-1]),t.rot90(3,[-2,-1]),\n              t.flip(-1).rot90(1,[-2,-1]),t.flip(-2).rot90(1,[-2,-1]),\n              t.flip(-1).rot90(2,[-2,-1]),t.flip(-2).rot90(2,[-2,-1]),\n              t.flip(-1).rot90(3,[-2,-1]),t.flip(-2).rot90(3,[-2,-1]),\n              t.roll(32,-1),t.roll(-32,-1),t.roll(32,-2),t.roll(-32,-2)]\n        return torch.stack([self.norm(a) for a in augs]),s\n\ndef tta_extract(model,ids,img_dir,dev,dl_kw):\n    model.eval(); loader=DataLoader(TTADS(ids,img_dir),**dl_kw)\n    ff,pp,ii=[],[],[]\n    with torch.no_grad():\n        for ab,sids in tqdm(loader,desc=\"TTA\",ncols=80):\n            B,N,C,H,W=ab.shape\n            fl=ab.view(B*N,C,H,W).to(dev)\n            ft=model(fl,ret=True); pr=torch.softmax(model.head(ft),-1)\n            ff.append(ft.view(B,N,-1).mean(1).cpu().numpy())\n            pp.append(pr.view(B,N,-1).mean(1).cpu().numpy())\n            ii.extend(list(sids))\n    return np.vstack(ff),np.vstack(pp),ii\n\nclass EffB3(nn.Module):\n    def __init__(self,unfreeze=5,n=9):\n        super().__init__()\n        bb=models.efficientnet_b3(weights=\"IMAGENET1K_V1\")\n        self.features=bb.features; self.avgpool=bb.avgpool\n        self.drop=nn.Dropout(0.35)\n        self.head=nn.Linear(3072,n)\n        for p in self.parameters(): p.requires_grad=False\n        for blk in list(self.features.children())[-unfreeze:]:\n            for p in blk.parameters(): p.requires_grad=True\n        for p in self.head.parameters(): p.requires_grad=True\n    def forward(self,x,ret=False):\n        x=self.features(x)\n        f=torch.cat([self.avgpool(x).flatten(1),F.adaptive_max_pool2d(x,1).flatten(1)],1)\n        f=self.drop(f)\n        return f if ret else self.head(f)\n\nclass ConvNeXtS(nn.Module):\n    \"\"\"ConvNeXt-Small: orthogonal architecture to EfficientNet -> diversity\"\"\"\n    def __init__(self,n=9):\n        super().__init__()\n        bb=models.convnext_small(weights=\"IMAGENET1K_V1\")\n        self.features=bb.features\n        self.avgpool =nn.AdaptiveAvgPool2d(1)\n        self.head    =nn.Linear(768,n)\n        for p in self.parameters(): p.requires_grad=False\n        # Unfreeze last 2 stages\n        for stage in list(self.features.children())[-2:]:\n            for p in stage.parameters(): p.requires_grad=True\n        for p in self.head.parameters(): p.requires_grad=True\n    def forward(self,x,ret=False):\n        f=self.avgpool(self.features(x)).flatten(1)  # (B,768)\n        return f if ret else self.head(f)\n\ndef mixup_data(x, y, alpha=0.2):\n    if alpha > 0: lam = np.random.beta(alpha, alpha)\n    else: lam = 1\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(x.device)\n    mixed_x = lam * x + (1 - lam) * x[index, :]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)\n\ndef train_cnn_5fold(ModelClass, model_name, feat_dim, img_dir, n_epochs=8, max_lr=1e-4):\n    FEAT_P = WORK_DIR/f\"features/cnn_{model_name}_feat_tr.npy\"\n    OOF_P  = WORK_DIR/f\"features/cnn_{model_name}_oof.npy\"\n    IDS_P  = WORK_DIR/f\"features/cnn_{model_name}_ids.pkl\"\n    FULL_P = WORK_DIR/f\"features/cnn_{model_name}_full.pth\"\n\n    # Bypass check if MODE == FULL\n    if MODE == \"FAST\":\n        for search in [FEAT_DIR, WORK_DIR/\"features\"]:\n            fp = search/f\"cnn_{model_name}_oof.npy\"\n            if fp.exists():\n                log.info(\"%s: loading precomputed from %s\", model_name, search)\n                feat_tr = np.load(search/f\"cnn_{model_name}_feat_tr.npy\", mmap_mode=\"r\")\n                oof_tr  = np.load(fp, mmap_mode=\"r\")\n                with open(search/f\"cnn_{model_name}_ids.pkl\",\"rb\") as f: ids_tr=pickle.load(f)\n                return feat_tr, oof_tr, ids_tr, search/f\"cnn_{model_name}_full.pth\"\n\n    label_map = dict(zip(df_labels[\"Id\"],df_labels[\"label\"]))\n    # ĐỔI THÀNH 3 FOLDS để kịp giờ Kaggle\n    skf_c = StratifiedKFold(n_splits=3,shuffle=True,random_state=SEED) \n    oof    = np.zeros((len(ALL_IDS),N_CLASSES),dtype=np.float32)\n    feat_buf=np.zeros((len(ALL_IDS),feat_dim), dtype=np.float32)\n    id2i   = {s:i for i,s in enumerate(ALL_IDS)}\n\n    for fold,(tri,vli) in enumerate(skf_c.split(ALL_IDS,df_labels[\"label\"].values)):\n        log.info(\"%s Fold %d/3\", model_name, fold+1)\n        tr_ids=[ALL_IDS[i] for i in tri]; val_ids=[ALL_IDS[i] for i in vli]\n        lm_tr={s:label_map[s] for s in tr_ids}\n        loader=DataLoader(MalwareDS(tr_ids,img_dir,lm_tr),**DL_TR)\n        model =ModelClass().to(DEVICE)\n        opt   =torch.optim.AdamW(filter(lambda p:p.requires_grad,model.parameters()), lr=1e-5,weight_decay=1e-4)\n        sch   =torch.optim.lr_scheduler.OneCycleLR(opt,max_lr=max_lr,steps_per_epoch=len(loader),epochs=n_epochs)\n        crit  =nn.CrossEntropyLoss(label_smoothing=0.05)\n        model.train()\n        for ep in range(n_epochs):\n            ls=0\n            for imgs,labs in tqdm(loader,desc=f\"{model_name} F{fold+1} E{ep+1}\",ncols=80):\n                imgs,labs=imgs.to(DEVICE),labs.to(DEVICE)\n                \n                # TÍCH HỢP MIXUP VÀO V7\n                imgs, labs_a, labs_b, lam = mixup_data(imgs, labs, alpha=0.3)\n                \n                opt.zero_grad()\n                preds = model(imgs)\n                l = mixup_criterion(crit, preds, labs_a, labs_b, lam)\n                \n                l.backward(); opt.step(); sch.step(); ls+=l.item()\n            log.info(\"  ep%d loss=%.4f\",ep+1,ls/len(loader))\n            \n        ft,pr,ids_v=tta_extract(model,val_ids,img_dir,DEVICE,DL_EX)\n        for s,f_,p_ in zip(ids_v,ft,pr):\n            oof[id2i[s]]=p_; feat_buf[id2i[s]]=f_\n        log.info(\"%s F%d OOF=%.5f\",model_name,fold+1,\n                 log_loss(df_labels.set_index(\"Id\").reindex(val_ids)[\"label\"].values,pr))\n        del model; torch.cuda.empty_cache(); gc.collect()\n\n    # Full retrain\n    log.info(\"%s: retraining on ALL data\", model_name)\n    full_loader=DataLoader(MalwareDS(ALL_IDS,img_dir,label_map),**DL_TR)\n    mf=ModelClass().to(DEVICE)\n    of=torch.optim.AdamW(filter(lambda p:p.requires_grad,mf.parameters()),lr=5e-6,weight_decay=1e-4)\n    sf=torch.optim.lr_scheduler.OneCycleLR(of,max_lr=max_lr*0.5,steps_per_epoch=len(full_loader),epochs=6)\n    cf=nn.CrossEntropyLoss(label_smoothing=0.02)\n    mf.train()\n    for ep in range(6): # Giảm epoch Full retrain xuống 6 để tiết kiệm giờ\n        ls=0\n        for imgs,labs in tqdm(full_loader,desc=f\"{model_name} Full E{ep+1}\",ncols=80):\n            imgs,labs=imgs.to(DEVICE),labs.to(DEVICE)\n            imgs, labs_a, labs_b, lam = mixup_data(imgs, labs, alpha=0.3)\n            of.zero_grad()\n            l = mixup_criterion(cf, mf(imgs), labs_a, labs_b, lam)\n            l.backward(); of.step(); sf.step(); ls+=l.item()\n        log.info(\"  full ep%d loss=%.4f\",ep+1,ls/len(full_loader))\n    torch.save(mf.state_dict(),FULL_P)\n    del mf; torch.cuda.empty_cache(); gc.collect()\n\n    np.save(FEAT_P,feat_buf); np.save(OOF_P,oof)\n    with open(IDS_P,\"wb\") as f: pickle.dump(ALL_IDS,f,protocol=4)\n    log.info(\"%s OOF loss=%.5f\",model_name,log_loss(df_labels[\"label\"].values,oof))\n    return feat_buf, oof, ALL_IDS, FULL_P\n\n# Train both CNNs\neffb3_feat_tr, effb3_oof, effb3_ids, effb3_pth = train_cnn_5fold(\n    EffB3, \"effb3\", 3072, IMG_DIR, n_epochs=10, max_lr=1e-4)\ncnvs_feat_tr,  cnvs_oof,  cnvs_ids,  cnvs_pth  = train_cnn_5fold(\n    ConvNeXtS, \"cnvs\", 768, IMG_DIR, n_epochs=10, max_lr=5e-5)\n\nprint(f\"EffB3 OOF: {log_loss(df_labels[\"label\"].values, effb3_oof):.5f}\")\nprint(f\"ConvNeXt  OOF: {log_loss(df_labels[\"label\"].values, cnvs_oof):.5f}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## E. Test Processing\n","metadata":{}},{"cell_type":"code","source":"TEST_TAB_P   = FEAT_DIR/\"tabular_features_test.csv\"\nTEST_IMG_DIR = WORK_DIR/\"test_images\"; TEST_IMG_DIR.mkdir(exist_ok=True)\n\ndef load_test_cnn(model_name, ModelClass, feat_dim, model_pth):\n    for search in [FEAT_DIR, WORK_DIR/\"features\"]:\n        fp=search/f\"cnn_{model_name}_feat_te.npy\"\n        if fp.exists():\n            ft=np.load(fp,mmap_mode=\"r\"); pr=np.load(search/f\"cnn_{model_name}_prob_te.npy\",mmap_mode=\"r\")\n            with open(search/f\"cnn_{model_name}_ids_te.pkl\",\"rb\") as f: ids=pickle.load(f)\n            log.info(\"%s test loaded: %s\", model_name, ft.shape); return ft,pr,ids\n    m=ModelClass().to(DEVICE)\n    m.load_state_dict(torch.load(model_pth,map_location=DEVICE))\n    ao=FileOps.get_archive_order(COMP_DIR/\"test.7z\")\n    eo=FileOps.sort_by_archive_order(TEST_IDS,ao)\n    ft,pr,ids=tta_extract(m,eo,TEST_IMG_DIR,DEVICE,DL_EX)\n    np.save(WORK_DIR/f\"features/cnn_{model_name}_feat_te.npy\",ft)\n    np.save(WORK_DIR/f\"features/cnn_{model_name}_prob_te.npy\",pr)\n    with open(WORK_DIR/f\"features/cnn_{model_name}_ids_te.pkl\",\"wb\") as f: pickle.dump(ids,f,protocol=4)\n    del m; torch.cuda.empty_cache(); gc.collect()\n    return ft,pr,ids\n\n# Tabular test\nif TEST_TAB_P.exists():\n    df_test_tab=pd.read_csv(TEST_TAB_P,index_col=\"Id\"); log.info(\"Test tab: %s\",df_test_tab.shape)\nelse:\n    ta=COMP_DIR/\"test.7z\"; ao=FileOps.get_archive_order(ta)\n    eo=FileOps.sort_by_archive_order(TEST_IDS,ao)\n    tbd=WORK_DIR/\"test_batches\"; CacheOps.clear_batch_records(tbd)\n    for bi,bids in enumerate(tqdm(list(FileOps.chunked(eo,cfg.batch_n)),desc=\"Test\")):\n        pairs=FileOps.extract_batch(ta,bids,cfg.scratch_dir); recs=[]\n        for sid,bp,ap in pairs:\n            try:\n                np.save(TEST_IMG_DIR/f\"{sid}.npy\",ImageOps.process_image(bp,ap,cfg))\n                r=OpcodeOps.parse_advanced(asm_path=ap,bytes_path=bp); r[\"id\"]=sid\n                bs=bp.stat().st_size if bp else 0; as_=ap.stat().st_size if ap else 0\n                r[\"meta\"]={\"bytes_size\":bs,\"asm_size\":as_,\"size_ratio\":as_/bs if bs>0 else 0.}\n                r[\"byte_entropy\"]=ByteEntropyOps.extract_all(bp) if bp else {}\n                r[\"markov2\"]=MarkovV2Ops.extract(bp) if bp else {}\n                r[\"asm_struct\"]=AsmStructOps.extract(ap) if ap else {}\n                recs.append(r)\n            except: pass\n            FileOps.delete_raw(bp,ap)\n        CacheOps.save_batch_records(recs,bi,tbd); gc.collect(); GPU.free()\n    df_test_tab=OpcodeOps.create_feature_df(CacheOps.iter_all_records(tbd),vocab,STAT_COLS)\n    for bn in [\"byte_entropy\",\"markov2\",\"asm_struct\"]:\n        rows={r[\"id\"]:r.get(bn,{}) for r in CacheOps.iter_all_records(tbd)}\n        dfe=pd.DataFrame.from_dict(rows,orient=\"index\",dtype=np.float32); dfe.index.name=\"Id\"\n        df_test_tab=df_test_tab.join(dfe,how=\"left\").fillna(0)\n    df_test_tab.to_csv(WORK_DIR/\"features/tabular_features_test.csv\")\n\nif all(c in df_test_tab.columns for c in META_COLS):\n    df_test_meta=df_test_tab[META_COLS].copy(); df_test_tab=df_test_tab.drop(columns=META_COLS)\nelse:\n    df_test_meta=pd.DataFrame(0,index=df_test_tab.index,columns=META_COLS)\n\neffb3_feat_te,effb3_prob_te,effb3_ids_te = load_test_cnn(\"effb3\",EffB3,3072,effb3_pth)\ncnvs_feat_te, cnvs_prob_te, cnvs_ids_te  = load_test_cnn(\"cnvs\", ConvNeXtS,768,cnvs_pth)\nprint(f\"Test tab={df_test_tab.shape} effb3={effb3_feat_te.shape} cnvs={cnvs_feat_te.shape}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## F. Assemble Feature Matrix\n","metadata":{}},{"cell_type":"code","source":"y_train = df_labels.set_index(\"Id\").reindex(ALL_IDS)[\"label\"].values\n\ndef align(feat,ids,ref_ids):\n    m={s:i for i,s in enumerate(ids)}\n    return np.array(feat)[[m[s] for s in ref_ids]]\n\n# Train: CNN features + probs aligned to ALL_IDS\nX_effb3_feat_tr = align(effb3_feat_tr, effb3_ids, ALL_IDS)   # (N,3072)\nX_effb3_prob_tr = align(effb3_oof,     effb3_ids, ALL_IDS)   # (N,9) OOF\nX_cnvs_feat_tr  = align(cnvs_feat_tr,  cnvs_ids,  ALL_IDS)   # (N,768)\nX_cnvs_prob_tr  = align(cnvs_oof,      cnvs_ids,  ALL_IDS)   # (N,9) OOF\nX_tab_tr        = df_tab.reindex(ALL_IDS).fillna(0).values\ndf_meta[\"unique_opcode_count\"] = (df_tab>0).sum(1)\nX_meta_tr = df_meta.reindex(ALL_IDS).fillna(0).values\n\n# Test: aligned to TEST_IDS\nX_effb3_feat_te = align(effb3_feat_te, effb3_ids_te, TEST_IDS)\nX_effb3_prob_te = align(effb3_prob_te, effb3_ids_te, TEST_IDS)\nX_cnvs_feat_te  = align(cnvs_feat_te,  cnvs_ids_te,  TEST_IDS)\nX_cnvs_prob_te  = align(cnvs_prob_te,  cnvs_ids_te,  TEST_IDS)\nX_tab_te  = df_test_tab.reindex(TEST_IDS,columns=df_tab.columns).fillna(0).values\n_opc_cols = [c for c in df_test_tab.columns if c not in META_COLS]\ndf_test_meta[\"unique_opcode_count\"] = (df_test_tab[_opc_cols]>0).sum(1)\nfor c in META_COLS:\n    if c not in df_test_meta.columns: df_test_meta[c]=0\nX_meta_te = df_test_meta.reindex(TEST_IDS,columns=df_meta.columns).fillna(0).values\n\n# Full matrix: [EffB3(3072)|CNvS(768)|EffB3-prob(9)|CNvS-prob(9)|tab(~6600)|meta(4)]\nX_train = np.hstack([X_effb3_feat_tr,X_cnvs_feat_tr,X_effb3_prob_tr,X_cnvs_prob_tr,X_tab_tr,X_meta_tr])\nX_test  = np.hstack([X_effb3_feat_te,X_cnvs_feat_te,X_effb3_prob_te,X_cnvs_prob_te,X_tab_te,X_meta_te])\nassert X_train.shape[1]==X_test.shape[1], f\"MISMATCH: {X_train.shape[1]} vs {X_test.shape[1]}\"\n\nlog.info(\"X_train=%s X_test=%s (effb3=%d cnvs=%d prob=%d tab=%d meta=%d)\",\n         X_train.shape,X_test.shape,X_effb3_feat_tr.shape[1],X_cnvs_feat_tr.shape[1],\n         X_effb3_prob_tr.shape[1]+X_cnvs_prob_tr.shape[1],X_tab_tr.shape[1],X_meta_tr.shape[1])\ndel X_effb3_feat_tr,X_cnvs_feat_tr,X_effb3_prob_tr,X_cnvs_prob_tr,X_tab_tr,X_meta_tr\ndel X_effb3_feat_te,X_cnvs_feat_te,X_effb3_prob_te,X_cnvs_prob_te,X_tab_te,X_meta_te\ngc.collect()\nprint(f\"X_train={X_train.shape}  X_test={X_test.shape}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## G. 10-Fold CV: XGB + LGB-gbdt + CB + ET\n","metadata":{}},{"cell_type":"code","source":"XGB_PARAMS={\n    \"objective\":\"multi:softprob\",\"num_class\":N_CLASSES,\"eval_metric\":\"mlogloss\",\n    \"max_depth\":10,\"learning_rate\":0.007,\"subsample\":0.85,\"colsample_bytree\":0.65,\n    \"min_child_weight\":5,\"gamma\":0.05,\"reg_alpha\":0.15,\"reg_lambda\":1.5,\n    \"tree_method\":\"hist\",\"device\":\"cuda\",\"seed\":SEED,\"verbosity\":0,\"max_bin\":255,\n}\n# V7: use gbdt (not DART) for LGB so early stopping works correctly\nLGB_PARAMS={\n    \"objective\":\"multiclass\",\"num_class\":N_CLASSES,\"metric\":\"multi_logloss\",\n    \"boosting_type\":\"gbdt\",\n    \"learning_rate\":0.007,\"num_leaves\":127,\"min_child_samples\":15,\n    \"subsample\":0.85,\"colsample_bytree\":0.65,\"feature_fraction\":0.65,\n    \"reg_alpha\":0.15,\"reg_lambda\":0.5,\"max_bin\":255,\"seed\":SEED,\"verbosity\":-1,\n}\nCB_PARAMS={\n    \"iterations\":4000,\"learning_rate\":0.015,\"depth\":10,\n    \"loss_function\":\"MultiClass\",\"eval_metric\":\"MultiClass\",\"task_type\":\"GPU\",\n    \"random_seed\":SEED,\"verbose\":300,\"early_stopping_rounds\":150,\"l2_leaf_reg\":5.0,\n    \"random_strength\":0.5,\"bagging_temperature\":0.5,\"border_count\":254,\n}\nET_PARAMS=dict(n_estimators=600,max_depth=None,min_samples_leaf=1,\n               n_jobs=-1,random_state=SEED,class_weight=\"balanced\")\n\n# Feature selection: top-2500 using RF + impurity\nif SEL_PATH.exists():\n    final_sel=np.load(SEL_PATH).tolist()\n    log.info(\"Feature sel loaded: %d\", len(final_sel))\nelse:\n    log.info(\"Feature selection via RF bootstrap 40%%...\")\n    rng=np.random.RandomState(SEED)\n    bi=rng.choice(len(y_train),int(len(y_train)*0.40),replace=False)\n    rf=RandomForestClassifier(n_estimators=400,max_depth=16,n_jobs=-1,random_state=SEED)\n    rf.fit(X_train[bi],y_train[bi])\n    final_sel=np.argsort(rf.feature_importances_)[::-1][:2500].tolist()\n    np.save(SEL_PATH,np.array(final_sel))\n    del rf; gc.collect()\n    log.info(\"Feature selection done: %d features\", len(final_sel))\n\nXtr_s=X_train[:,final_sel]; Xte_s=X_test[:,final_sel]\nsw=compute_sample_weight(\"balanced\",y_train)\nskf=StratifiedKFold(n_splits=N_FOLDS,shuffle=True,random_state=SEED)\n\noof_xgb=np.zeros((len(y_train),N_CLASSES)); tst_xgb=np.zeros((len(TEST_IDS),N_CLASSES))\noof_lgb=np.zeros((len(y_train),N_CLASSES)); tst_lgb=np.zeros((len(TEST_IDS),N_CLASSES))\noof_cb =np.zeros((len(y_train),N_CLASSES)); tst_cb =np.zeros((len(TEST_IDS),N_CLASSES))\noof_et =np.zeros((len(y_train),N_CLASSES)); tst_et =np.zeros((len(TEST_IDS),N_CLASSES))\nfm_xgb,fm_lgb,fm_cb=[],[],[]\nfold_scores=[]; t0_cv=time.time()\n\nfor fold,(tri,vli) in enumerate(skf.split(Xtr_s,y_train)):\n    t0=time.time()\n    Xtr,Xvl=Xtr_s[tri],Xtr_s[vli]\n    ytr,yvl=y_train[tri],y_train[vli]\n    wtr=sw[tri]\n\n    mx=xgb.train(XGB_PARAMS,xgb.DMatrix(Xtr,label=ytr,weight=wtr),\n                 num_boost_round=4000,evals=[(xgb.DMatrix(Xvl,label=yvl),\"v\")],\n                 early_stopping_rounds=100,verbose_eval=400)\n    vx=mx.predict(xgb.DMatrix(Xvl))\n    oof_xgb[vli]=vx; tst_xgb+=mx.predict(xgb.DMatrix(Xte_s))/N_FOLDS; fm_xgb.append(mx)\n\n    dtl=lgb.Dataset(Xtr,label=ytr,weight=wtr); dvl=lgb.Dataset(Xvl,label=yvl,reference=dtl)\n    ml=lgb.train(LGB_PARAMS,dtl,num_boost_round=4000,valid_sets=[dvl],\n                 callbacks=[lgb.early_stopping(100),lgb.log_evaluation(400)])\n    vl=ml.predict(Xvl,num_iteration=ml.best_iteration)\n    oof_lgb[vli]=vl; tst_lgb+=ml.predict(Xte_s,num_iteration=ml.best_iteration)/N_FOLDS; fm_lgb.append(ml)\n\n    mc=CatBoostClassifier(**CB_PARAMS)\n    mc.fit(Pool(Xtr,label=ytr,weight=wtr),eval_set=Pool(Xvl,label=yvl),use_best_model=True)\n    vc=mc.predict_proba(Pool(Xvl,label=yvl))\n    oof_cb[vli]=vc; tst_cb+=mc.predict_proba(Xte_s)/N_FOLDS; fm_cb.append(mc)\n\n    me=ExtraTreesClassifier(**ET_PARAMS); me.fit(Xtr,ytr,sample_weight=wtr)\n    ve=me.predict_proba(Xvl)\n    oof_et[vli]=ve; tst_et+=me.predict_proba(Xte_s)/N_FOLDS\n\n    lx,ll,lc,le=log_loss(yvl,vx),log_loss(yvl,vl),log_loss(yvl,vc),log_loss(yvl,ve)\n    fold_scores.append({\"xgb\":lx,\"lgb\":ll,\"cb\":lc,\"et\":le})\n    log.info(\"Fold%2d xgb=%.4f lgb=%.4f cb=%.4f et=%.4f | %.1fs\",fold+1,lx,ll,lc,le,time.time()-t0)\n\nlog.info(\"CV done %.1f min\", (time.time()-t0_cv)/60)\nprint(\"Individual OOF:\")\nfor name,oof in [(\"XGB\",oof_xgb),(\"LGB\",oof_lgb),(\"CB\",oof_cb),(\"ET\",oof_et)]:\n    print(f\"  {name}: {log_loss(y_train,oof):.6f}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## H. Per-class Isotonic Calibration + Nelder-Mead Ensemble\n","metadata":{}},{"cell_type":"code","source":"def isotonic_calibrate(oof_probs, tst_probs, y_train, n_classes=9):\n    \"\"\"V7: Per-class isotonic regression calibration.\n    Significantly better calibrated probabilities than temperature scaling.\n    \"\"\"\n    cal_test = np.zeros_like(tst_probs)\n    for c in range(n_classes):\n        y_bin = (y_train == c).astype(float)\n        ir = IsotonicRegression(out_of_bounds=\"clip\")\n        ir.fit(oof_probs[:, c], y_bin)\n        cal_test[:, c] = ir.predict(tst_probs[:, c])\n    # Renormalize\n    s = cal_test.sum(1, keepdims=True); s[s==0]=1\n    return cal_test / s\n\n# Calibrate each model\ntst_xgb_cal = isotonic_calibrate(oof_xgb, tst_xgb, y_train)\ntst_lgb_cal = isotonic_calibrate(oof_lgb, tst_lgb, y_train)\ntst_cb_cal  = isotonic_calibrate(oof_cb,  tst_cb,  y_train)\ntst_et_cal  = isotonic_calibrate(oof_et,  tst_et,  y_train)\nlog.info(\"Isotonic calibration done\")\n\n# Nelder-Mead over 4 models (weights only, calibration already done)\ndef ens_loss(w4):\n    w=np.abs(w4); w/=w.sum()\n    return log_loss(y_train,sum(wi*oi for wi,oi in zip(w,[oof_xgb,oof_lgb,oof_cb,oof_et])))\n\nopt=minimize(ens_loss,[.25,.25,.25,.25],method=\"Nelder-Mead\",\n             options={\"maxiter\":3000,\"xatol\":1e-9,\"fatol\":1e-9})\nopt_w=np.abs(opt.x); opt_w/=opt_w.sum()\n\noof_ens=sum(wi*oi for wi,oi in zip(opt_w,[oof_xgb,oof_lgb,oof_cb,oof_et]))\ntst_ens=sum(wi*ti for wi,ti in zip(opt_w,[tst_xgb_cal,tst_lgb_cal,tst_cb_cal,tst_et_cal]))\nens_loss_val=log_loss(y_train,oof_ens)\n\nprint(f\"L1 ensemble OOF loss: {ens_loss_val:.6f}\")\nprint(f\"Weights: XGB={opt_w[0]:.3f} LGB={opt_w[1]:.3f} CB={opt_w[2]:.3f} ET={opt_w[3]:.3f}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## I. Level-2: XGBoost Meta-learner (5-fold OOF)\n\n**V7**: XGBoost meta replaces MLP — better regularization for 63-dim input\n","metadata":{}},{"cell_type":"code","source":"# Stack: 4 GBDT OOF (36) + EffB3 OOF (9) + ConvNeXt OOF (9) = 54 features\nX_l2_tr = np.hstack([oof_xgb, oof_lgb, oof_cb, oof_et, X_effb3_prob_tr, X_cnvs_prob_tr])  # (N,54)\nX_l2_te = np.hstack([tst_xgb_cal, tst_lgb_cal, tst_cb_cal, tst_et_cal,\n                     align(effb3_prob_te,effb3_ids_te,TEST_IDS),\n                     align(cnvs_prob_te, cnvs_ids_te, TEST_IDS)])  # (M,54)\n\n# Re-load CNN probs for test (may have been freed)\ntry: _ = X_l2_te.shape\nexcept NameError:\n    effb3_prob_te=np.load(WORK_DIR/\"features/cnn_effb3_prob_te.npy\",mmap_mode=\"r\")\n    cnvs_prob_te =np.load(WORK_DIR/\"features/cnn_cnvs_prob_te.npy\", mmap_mode=\"r\")\n\nXGB_META={\n    \"objective\":\"multi:softprob\",\"num_class\":N_CLASSES,\"eval_metric\":\"mlogloss\",\n    \"max_depth\":4,\"learning_rate\":0.02,\"subsample\":0.8,\"colsample_bytree\":0.8,\n    \"reg_alpha\":1.0,\"reg_lambda\":2.0,\n    \"tree_method\":\"hist\",\"device\":\"cuda\",\"seed\":SEED,\"verbosity\":0,\n}\n\nskf_l2=StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED+77)\noof_meta=np.zeros((len(y_train),N_CLASSES))\ntst_meta=np.zeros((len(TEST_IDS),N_CLASSES))\n\nfor fold,(tr,vl) in enumerate(skf_l2.split(X_l2_tr,y_train)):\n    mx=xgb.train(XGB_META,xgb.DMatrix(X_l2_tr[tr],label=y_train[tr]),\n                 num_boost_round=500,\n                 evals=[(xgb.DMatrix(X_l2_tr[vl],label=y_train[vl]),\"v\")],\n                 early_stopping_rounds=30,verbose_eval=0)\n    oof_meta[vl]=mx.predict(xgb.DMatrix(X_l2_tr[vl]))\n    tst_meta+=mx.predict(xgb.DMatrix(X_l2_te))/5\n    log.info(\"Meta fold %d | val=%.5f\",fold+1,log_loss(y_train[vl],oof_meta[vl]))\n\nmeta_loss=log_loss(y_train,oof_meta)\nlog.info(\"L2 XGB-Meta OOF=%.6f  (L1=%.6f)\", meta_loss, ens_loss_val)\n\n# Optimal blend of L1 + L2\nbest_loss=float(\"inf\"); best_a=1.0\nfor a in np.linspace(0,1,41):\n    b=a*oof_meta+(1-a)*oof_ens\n    l=log_loss(y_train,b)\n    if l<best_loss: best_loss=l; best_a=a\n\nlog.info(\"Best blend: alpha=%.2f  final_OOF=%.6f\", best_a, best_loss)\ntest_preds_final = best_a*tst_meta + (1-best_a)*tst_ens\noof_final        = best_a*oof_meta + (1-best_a)*oof_ens\nprint(f\"Final OOF: loss={best_loss:.6f}  acc={accuracy_score(y_train,oof_final.argmax(1)):.4f}\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## J. Pseudo-labeling (threshold 0.999, 3 rounds)\n","metadata":{}},{"cell_type":"code","source":"PSEUDO_THRESH, MAX_RDS = 0.999, 3\nXaug,yaug,waug = Xtr_s.copy(),y_train.copy(),sw.copy()\ncur_loss = best_loss\n\nfor rnd in range(MAX_RDS):\n    mask = test_preds_final.max(1) >= PSEUDO_THRESH\n    np_ = mask.sum()\n    log.info(\"Pseudo %d: %d/%d samples (thresh=%.3f)\", rnd+1, np_, len(TEST_IDS), PSEUDO_THRESH)\n    if np_ == 0: PSEUDO_THRESH -= 0.002; continue\n\n    yp  = test_preds_final[mask].argmax(1)\n    wp  = compute_sample_weight(\"balanced\",np.concatenate([yaug,yp]))[len(yaug):]\n    Xaug=np.vstack([Xaug,Xte_s[mask]]); yaug=np.concatenate([yaug,yp]); waug=np.concatenate([waug,wp])\n    log.info(\"Augmented: %d samples\", len(yaug))\n\n    rx=int(np.mean([m.best_iteration for m in fm_xgb]))\n    rl=int(np.mean([m.best_iteration for m in fm_lgb]))\n    rc=int(np.mean([m.best_iteration_ for m in fm_cb]))\n\n    px=xgb.train({**XGB_PARAMS,\"verbosity\":0},xgb.DMatrix(Xaug,label=yaug,weight=waug),num_boost_round=rx)\n    pl=lgb.train({**LGB_PARAMS,\"verbose\":-1},lgb.Dataset(Xaug,label=yaug,weight=waug),num_boost_round=rl)\n    pc=CatBoostClassifier(**{**CB_PARAMS,\"iterations\":rc,\"verbose\":0})\n    pc.fit(Pool(Xaug,label=yaug,weight=waug))\n    pe=ExtraTreesClassifier(**ET_PARAMS); pe.fit(Xaug,yaug,sample_weight=waug)\n\n    new_tst_ens=sum(wi*isotonic_calibrate(oof_oi,tp,y_train)\n                    for wi,tp,oof_oi in zip(opt_w,[\n                        px.predict(xgb.DMatrix(Xte_s)),\n                        pl.predict(Xte_s,num_iteration=rl),\n                        pc.predict_proba(Xte_s),\n                        pe.predict_proba(Xte_s)\n                    ],[oof_xgb,oof_lgb,oof_cb,oof_et]))\n    new_final = best_a*tst_meta + (1-best_a)*new_tst_ens\n\n    # Mini-CV to check improvement\n    skf3=StratifiedKFold(3,shuffle=True,random_state=SEED+3)\n    p_oof=np.zeros((len(y_train),N_CLASSES))\n    for ptr,pvl in skf3.split(Xaug[:len(y_train)],yaug[:len(y_train)]):\n        _m=xgb.train({**XGB_PARAMS,\"verbosity\":0},\n                     xgb.DMatrix(Xaug[ptr],label=yaug[ptr],weight=waug[ptr]),num_boost_round=rx)\n        p_oof[pvl]=_m.predict(xgb.DMatrix(Xaug[pvl]))\n    nl=log_loss(y_train,p_oof)\n    log.info(\"Pseudo %d: new_loss=%.6f (was %.6f)\", rnd+1, nl, cur_loss)\n    if nl < cur_loss-0.0002:\n        test_preds_final=new_final; cur_loss=nl; log.info(\"Pseudo improved\")\n    else:\n        PSEUDO_THRESH-=0.001; log.info(\"No improvement — lower thresh to %.3f\", PSEUDO_THRESH)\n        if PSEUDO_THRESH < 0.990: break\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## K. Per-class Analysis + All Submissions\n","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\noof_cls=oof_final.argmax(1)\nprint(classification_report(y_train,oof_cls,target_names=[CLASS_NAMES[i+1] for i in range(9)]))\nprint(\"Per-class:\")\nfor c in range(9):\n    mk=y_train==c\n    if mk.sum():\n        acc=(oof_cls[mk]==c).mean()\n        flag=\"[OK]\" if acc>=0.999 else (\"[~]\" if acc>=0.99 else \"[X]\")\n        print(f\"  {flag} Class{c+1} ({CLASS_NAMES[c+1]:<18}) {acc:.5f} n={mk.sum()}\")\n\ndef sub(preds,fname,oof_v):\n    df=pd.DataFrame(preds,columns=[f\"Prediction{i}\" for i in range(1,10)])\n    df.insert(0,\"Id\",TEST_IDS); df.to_csv(WORK_DIR/fname,index=False)\n    log.info(\"Saved %-50s OOF=%.6f\",fname,oof_v)\n\nsub(test_preds_final,              \"submission_v7_FINAL.csv\",         log_loss(y_train,oof_final))\nsub(isotonic_calibrate(oof_xgb,tst_xgb,y_train), \"submission_v7_xgb.csv\",  log_loss(y_train,oof_xgb))\nsub(isotonic_calibrate(oof_lgb,tst_lgb,y_train), \"submission_v7_lgb.csv\",  log_loss(y_train,oof_lgb))\nsub(isotonic_calibrate(oof_cb, tst_cb, y_train), \"submission_v7_cb.csv\",   log_loss(y_train,oof_cb))\nsub(tst_et,                        \"submission_v7_et.csv\",            log_loss(y_train,oof_et))\nsub(tst_meta,                      \"submission_v7_l2meta.csv\",        log_loss(y_train,oof_meta))\nsub(tst_ens,                       \"submission_v7_l1ens.csv\",         ens_loss_val)\n\nS=\"=\"*65\nprint(f\"\\n{S}\\n  FINAL RESULTS — V7 ULTIMATE\\n{S}\")\nrows=[\n    (\"XGBoost\",        log_loss(y_train,oof_xgb),  \"submission_v7_xgb.csv\"),\n    (\"LightGBM\",       log_loss(y_train,oof_lgb),  \"submission_v7_lgb.csv\"),\n    (\"CatBoost\",       log_loss(y_train,oof_cb),   \"submission_v7_cb.csv\"),\n    (\"ExtraTrees\",     log_loss(y_train,oof_et),   \"submission_v7_et.csv\"),\n    (\"L1 Ensemble\",    ens_loss_val,               \"submission_v7_l1ens.csv\"),\n    (\"L2 XGB-Meta\",    meta_loss,                  \"submission_v7_l2meta.csv\"),\n    (\"V7 FINAL\",       log_loss(y_train,oof_final),\"submission_v7_FINAL.csv  <<< SUBMIT\"),\n]\nfor n,l,f in rows: print(f\"  {n:<18} {l:.6f}   {f}\")\nprint(f\"{S}\")\nprint(f\"  EffB3 OOF     : {log_loss(y_train,effb3_oof):.6f}\")\nprint(f\"  ConvNeXt OOF  : {log_loss(y_train,cnvs_oof):.6f}\")\nprint(f\"  Feature dim   : {X_train.shape[1]} raw -> {len(final_sel)} selected\")\nprint(f\"  Blend alpha   : {best_a:.2f} (L2-meta) + {1-best_a:.2f} (L1-ens)\")\nprint(f\"{S}\")\n","metadata":{},"outputs":[],"execution_count":null}]}