{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15918729,"datasetId":10208893,"databundleVersionId":16875168}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys, gc, time, logging, zipfile, shutil\nimport numpy as np, pandas as pd\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\n\n# 1. SETUP ĐƯỜNG DẪN \nCOMP_DIR   = Path(\"/kaggle/input/competitions/malware-classification\")\n# Nhớ nhắc bạn ấy đổi đường dẫn UTILS_PATH cho đúng với dataset tiện ích\nUTILS_PATH = Path(\"/kaggle/input/datasets/ngovietan/malware-utils-v7\")\nif str(UTILS_PATH) not in sys.path: sys.path.append(str(UTILS_PATH))\nfrom malware_utils_v7 import (Config, GPU, FileOps, ImageOps, OpcodeOps,\n                               CacheOps, ByteEntropyOps, MarkovV2Ops, AsmStructOps)\n\nlogging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\nlog = logging.getLogger(\"test_extract\")\n\nWORK_DIR = Path(\"/kaggle/working\")\ncfg = Config(base_dir=COMP_DIR, batch_n=80, min_free_gb=5.0)\ncfg.make_dirs(); GPU.setup()\n\n# 2. LẤY ID CỦA TẬP TEST\nTEST_IDS = pd.read_csv(COMP_DIR/\"sampleSubmission.csv\")[\"Id\"].tolist()\nTEST_IMG_DIR = WORK_DIR/\"test_images\"; TEST_IMG_DIR.mkdir(exist_ok=True)\nTEST_BATCH_DIR = WORK_DIR/\"test_batches\"; CacheOps.clear_batch_records(TEST_BATCH_DIR)\n\nta = COMP_DIR/\"test.7z\"\nao = FileOps.get_archive_order(ta)\neo = FileOps.sort_by_archive_order(TEST_IDS, ao)\nbatches = list(FileOps.chunked(eo, cfg.batch_n))\n\n# 3. HÀM TRÍCH XUẤT FULL TÍNH NĂNG V7\ndef extract_v7(sid, bp, ap):\n    c = OpcodeOps.parse_advanced(asm_path=ap, bytes_path=bp)\n    c[\"id\"] = sid\n    bs  = bp.stat().st_size if bp else 0\n    as_ = ap.stat().st_size if ap else 0\n    c[\"meta\"] = {\"bytes_size\":bs,\"asm_size\":as_,\"size_ratio\":as_/bs if bs>0 else 0.}\n    c[\"byte_entropy\"] = ByteEntropyOps.extract_all(bp) if bp else {}\n    c[\"markov2\"]      = MarkovV2Ops.extract(bp)        if bp else {}\n    c[\"asm_struct\"]   = AsmStructOps.extract(ap)       if ap else {}\n    return c\n\n# 4. CHẠY VÒNG LẶP SƠ CHẾ\nt0 = time.time()\nfor bi, bids in enumerate(batches):\n    if FileOps.get_free_gb() < cfg.min_free_gb: break\n    log.info(\"Test Batch %03d/%03d\", bi+1, len(batches))\n    pairs = FileOps.extract_batch(ta, bids, cfg.scratch_dir)\n    recs = []\n    for sid, bp, ap in pairs:\n        try:\n            # Lưu ảnh\n            np.save(TEST_IMG_DIR/f\"{sid}.npy\", ImageOps.process_image(bp,ap,cfg))\n            # Lưu dữ liệu Opcodes thô\n            recs.append(extract_v7(sid, bp, ap))\n        except Exception as e: log.debug(\"Fail %s: %s\", sid, e)\n        FileOps.delete_raw(bp, ap)\n    CacheOps.save_batch_records(recs, bi, TEST_BATCH_DIR)\n    gc.collect(); GPU.free()\n    shutil.rmtree(cfg.scratch_dir, ignore_errors=True); cfg.scratch_dir.mkdir(exist_ok=True)\n\n# 5. ĐÓNG GÓI SẢN PHẨM\nlog.info(\"Đang nén ảnh...\")\nwith zipfile.ZipFile(WORK_DIR/\"test_image_features.zip\", \"w\", compression=zipfile.ZIP_STORED, allowZip64=True) as zf:\n    for p in tqdm(list(TEST_IMG_DIR.glob(\"*.npy\"))): zf.write(p, arcname=p.name)\n\nlog.info(\"Đang nén dữ liệu thô...\")\nwith zipfile.ZipFile(WORK_DIR/\"test_batch_records.zip\", \"w\", compression=zipfile.ZIP_DEFLATED) as zf:\n    for p in list(TEST_BATCH_DIR.glob(\"*.pkl\")): zf.write(p, arcname=p.name)\n\nlog.info(\"HOÀN TẤT TẬP TEST! Thời gian: %.1f min\", (time.time()-t0)/60)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}