{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-30T20:20:20.624027Z","iopub.execute_input":"2026-03-30T20:20:20.625154Z","iopub.status.idle":"2026-03-30T20:20:20.972214Z","shell.execute_reply.started":"2026-03-30T20:20:20.625110Z","shell.execute_reply":"2026-03-30T20:20:20.971149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Kaggle Notebook: BIG 2015 ASM Opcode Tokenization Comparison\n# Inputs:\n#   /kaggle/input/competitions/malware-classification/dataSample.7z\n#   /kaggle/input/competitions/malware-classification/trainLabels.csv\n#   /kaggle/input/competitions/malware-classification/sampleSubmission.csv\n#   /kaggle/input/competitions/malware-classification/test.7z\n#   /kaggle/input/competitions/malware-classification/train.7z\n#\n# What this notebook does:\n#   1) Reads trainLabels.csv\n#   2) Randomly samples a small set of malware IDs\n#   3) Extracts only the selected .asm files from train.7z\n#   4) Parses opcode mnemonics from ASM text\n#   5) Groups opcodes into action categories\n#   6) Runs NSF-style tokenization:\n#        - Word2Vec on tokenized documents\n#        - mean-pooled vector per sample\n#   7) Runs Tokenization-Testing-style tokenization:\n#        - unigrams\n#        - bigrams\n#        - top-vocabulary filtering\n#   8) Saves outputs and comparison tables\n#\n# Notes:\n#   - This competition is malware-only, not malware-vs-goodware.\n#   - We use .asm files, not .bytes files.\n#   - We do NOT extract the full archive; only selected samples.\n# ============================================================\n\n# -----------------------------\n# 1. Install / import packages\n# -----------------------------\nimport os\nimport re\nimport random\nimport warnings\nfrom collections import Counter\n\nimport numpy as np\nimport pandas as pd\n\nwarnings.filterwarnings(\"ignore\")\n\ntry:\n    import py7zr\nexcept ImportError:\n    !pip -q install py7zr\n    import py7zr\n\ntry:\n    from gensim.models import Word2Vec\nexcept ImportError:\n    !pip -q install gensim\n    from gensim.models import Word2Vec\n\n# -----------------------------\n# 2. Config\n# -----------------------------\nTRAIN_7Z = \"/kaggle/input/competitions/malware-classification/train.7z\"\nTRAIN_LABELS = \"/kaggle/input/competitions/malware-classification/trainLabels.csv\"\n\nWORK_DIR = \"/kaggle/working\"\nEXTRACT_DIR = os.path.join(WORK_DIR, \"asm_extract\")\n\nRANDOM_SEED = 42\n\n# Since this dataset is malware-only, sample malware only here.\n# Change this to 40 or another number as needed.\nNUM_SAMPLES = 40\n\nVECTOR_SIZE = 104\nWINDOW_SIZE = 10\nMIN_COUNT = 1\n\nTOP_UNIGRAMS = 31\nTOP_BIGRAMS = 30\n\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\n\nos.makedirs(EXTRACT_DIR, exist_ok=True)\n\n# -----------------------------\n# 3. Load labels\n# -----------------------------\nif not os.path.exists(TRAIN_LABELS):\n    raise FileNotFoundError(f\"Missing labels file: {TRAIN_LABELS}\")\n\nlabels_df = pd.read_csv(TRAIN_LABELS)\n\n# Expected columns in this competition are usually Id and Class\nrequired_cols = {\"Id\", \"Class\"}\nmissing_cols = required_cols - set(labels_df.columns)\nif missing_cols:\n    raise ValueError(f\"Missing required columns in trainLabels.csv: {missing_cols}\")\n\nprint(\"Labels shape:\", labels_df.shape)\ndisplay(labels_df.head())\n\n# -----------------------------\n# 4. Sample a small malware subset\n# -----------------------------\nif len(labels_df) < NUM_SAMPLES:\n    raise ValueError(f\"Requested {NUM_SAMPLES} samples but only found {len(labels_df)} rows.\")\n\nsampled_labels = labels_df.sample(n=NUM_SAMPLES, random_state=RANDOM_SEED).reset_index(drop=True)\nsampled_ids = sampled_labels[\"Id\"].astype(str).tolist()\n\nprint(f\"Sampled {len(sampled_ids)} malware IDs\")\ndisplay(sampled_labels.head())\n\n# -----------------------------\n# 5. Inspect archive contents and extract selected .asm files only\n# -----------------------------\nif not os.path.exists(TRAIN_7Z):\n    raise FileNotFoundError(f\"Missing archive: {TRAIN_7Z}\")\n\nwith py7zr.SevenZipFile(TRAIN_7Z, mode=\"r\") as z:\n    archive_names = z.getnames()\n\nprint(f\"Archive contains {len(archive_names)} files\")\n\n# Build a lookup from sample Id -> archive member path for .asm\n# Handles either \"xxx.asm\" or nested paths like \"train/xxx.asm\"\nid_to_member = {}\n\narchive_set = set(archive_names)\nfor sample_id in sampled_ids:\n    direct_name = f\"{sample_id}.asm\"\n    if direct_name in archive_set:\n        id_to_member[sample_id] = direct_name\n        continue\n\n    # fallback: suffix match\n    matches = [name for name in archive_names if name.endswith(f\"/{sample_id}.asm\") or name.endswith(f\"\\\\{sample_id}.asm\")]\n    if matches:\n        id_to_member[sample_id] = matches[0]\n\nmissing_asm = [sid for sid in sampled_ids if sid not in id_to_member]\nif missing_asm:\n    print(\"Warning: could not locate .asm files for these IDs:\")\n    print(missing_asm[:20])\n\nextract_targets = list(id_to_member.values())\nprint(f\"Extracting {len(extract_targets)} .asm files...\")\n\nwith py7zr.SevenZipFile(TRAIN_7Z, mode=\"r\") as z:\n    z.extract(path=EXTRACT_DIR, targets=extract_targets)\n\nprint(\"Extraction complete.\")\n\n# -----------------------------\n# 6. Locate extracted .asm files\n# -----------------------------\ndef find_extracted_file(root_dir, filename):\n    for root, _, files in os.walk(root_dir):\n        if filename in files:\n            return os.path.join(root, filename)\n    return None\n\nsampled_labels[\"asm_path\"] = sampled_labels[\"Id\"].astype(str).apply(lambda x: find_extracted_file(EXTRACT_DIR, f\"{x}.asm\"))\n\navailable_df = sampled_labels.dropna(subset=[\"asm_path\"]).copy().reset_index(drop=True)\n\nprint(f\"Available extracted ASM files: {len(available_df)} / {len(sampled_labels)}\")\ndisplay(available_df.head())\n\nif len(available_df) == 0:\n    raise ValueError(\"No .asm files were extracted/found.\")\n\n# -----------------------------\n# 7. Parse opcode mnemonics from .asm files\n# -----------------------------\n# We keep only the mnemonic (instruction word), roughly matching your professor's guidance.\n# Typical ASM lines may look like:\n#   .text:00401000 mov eax, ebx\n#   .text:00401005 call sub_401020\n#\n# Heuristic:\n#   - look for code-section-like lines\n#   - strip addresses/labels\n#   - capture the first opcode-like token after them\n\nVALID_OPCODE_RE = re.compile(r\"^[a-z][a-z0-9_.]*$\")\n\nCOMMON_NON_OPCODES = {\n    \"db\", \"dd\", \"dw\", \"dq\", \"dt\", \"align\", \"assume\", \"end\", \"ends\",\n    \"extrn\", \"public\", \"proc\", \"endp\", \"segment\", \"byte\", \"word\", \"dword\",\n    \"qword\", \"large\", \"small\", \"near\", \"far\", \"offset\", \"ptr\", \"text\",\n    \"data\", \"rdata\", \"idata\", \"code\"\n}\n\ndef extract_opcodes_from_asm_file(filepath):\n    opcodes = []\n\n    with open(filepath, \"r\", encoding=\"latin1\", errors=\"ignore\") as f:\n        for line in f:\n            line = line.strip().lower()\n            if not line:\n                continue\n\n            # Skip obvious non-code/comment-ish lines\n            if line.startswith(\";\"):\n                continue\n\n            # Replace tabs/commas with spaces\n            clean = re.sub(r\"[\\t,]+\", \" \", line)\n\n            # Split into tokens\n            tokens = clean.split()\n            if len(tokens) < 2:\n                continue\n\n            # Try to find the mnemonic after an address/label token\n            # Example token patterns:\n            #   .text:00401000 mov eax ebx\n            #   text:00401000 mov eax ebx\n            #   00401000 mov eax ebx\n            #\n            mnemonic = None\n\n            for i, tok in enumerate(tokens[:-1]):\n                next_tok = tokens[i + 1]\n\n                # case 1: current token looks like address/section label\n                if \":\" in tok or re.fullmatch(r\"[0-9a-f]{4,}\", tok):\n                    if VALID_OPCODE_RE.match(next_tok):\n                        mnemonic = next_tok\n                        break\n\n            # fallback: sometimes first token itself might be mnemonic\n            if mnemonic is None:\n                first_tok = tokens[0]\n                if VALID_OPCODE_RE.match(first_tok):\n                    mnemonic = first_tok\n\n            if mnemonic is None:\n                continue\n\n            if mnemonic in COMMON_NON_OPCODES:\n                continue\n\n            opcodes.append(mnemonic)\n\n    return opcodes\n\navailable_df[\"raw_tokens\"] = available_df[\"asm_path\"].apply(extract_opcodes_from_asm_file)\navailable_df = available_df[available_df[\"raw_tokens\"].map(len) > 0].reset_index(drop=True)\n\nprint(\"Samples with non-empty opcode sequences:\", len(available_df))\ndisplay(available_df[[\"Id\", \"Class\", \"asm_path\"]].head())\n\n# -----------------------------\n# 8. Define opcode action categories\n# -----------------------------\n# Overlap fix:\n#   push/pop -> STACK\n#   test     -> COMPARE\n\nopcode_to_group = {\n    # MOVE\n    \"mov\": \"MOVE\",\n    \"lea\": \"MOVE\",\n    \"xchg\": \"MOVE\",\n\n    # ARITH\n    \"add\": \"ARITH\",\n    \"sub\": \"ARITH\",\n    \"mul\": \"ARITH\",\n    \"imul\": \"ARITH\",\n    \"div\": \"ARITH\",\n    \"idiv\": \"ARITH\",\n    \"inc\": \"ARITH\",\n    \"dec\": \"ARITH\",\n    \"adc\": \"ARITH\",\n    \"sbb\": \"ARITH\",\n    \"neg\": \"ARITH\",\n\n    # LOGIC\n    \"and\": \"LOGIC\",\n    \"or\": \"LOGIC\",\n    \"xor\": \"LOGIC\",\n    \"not\": \"LOGIC\",\n\n    # COMPARE\n    \"cmp\": \"COMPARE\",\n    \"test\": \"COMPARE\",\n\n    # JUMP\n    \"jmp\": \"JUMP\",\n    \"je\": \"JUMP\",\n    \"jne\": \"JUMP\",\n    \"jg\": \"JUMP\",\n    \"jge\": \"JUMP\",\n    \"jl\": \"JUMP\",\n    \"jle\": \"JUMP\",\n    \"ja\": \"JUMP\",\n    \"jae\": \"JUMP\",\n    \"jb\": \"JUMP\",\n    \"jbe\": \"JUMP\",\n    \"jnz\": \"JUMP\",\n    \"jz\": \"JUMP\",\n    \"jo\": \"JUMP\",\n    \"jno\": \"JUMP\",\n    \"js\": \"JUMP\",\n    \"jns\": \"JUMP\",\n\n    # CALL\n    \"call\": \"CALL\",\n    \"ret\": \"CALL\",\n    \"retn\": \"CALL\",\n    \"retf\": \"CALL\",\n\n    # STACK\n    \"push\": \"STACK\",\n    \"pop\": \"STACK\",\n    \"enter\": \"STACK\",\n    \"leave\": \"STACK\",\n    \"pusha\": \"STACK\",\n    \"popa\": \"STACK\",\n    \"pushad\": \"STACK\",\n    \"popad\": \"STACK\",\n\n    # STRING/MEMORY\n    \"stos\": \"STRING_MEMORY\",\n    \"stosb\": \"STRING_MEMORY\",\n    \"stosd\": \"STRING_MEMORY\",\n    \"lods\": \"STRING_MEMORY\",\n    \"lodsb\": \"STRING_MEMORY\",\n    \"lodsd\": \"STRING_MEMORY\",\n    \"movs\": \"STRING_MEMORY\",\n    \"movsb\": \"STRING_MEMORY\",\n    \"movsd\": \"STRING_MEMORY\",\n    \"cmps\": \"STRING_MEMORY\",\n    \"cmpsb\": \"STRING_MEMORY\",\n    \"cmpsd\": \"STRING_MEMORY\",\n    \"scas\": \"STRING_MEMORY\",\n    \"scasb\": \"STRING_MEMORY\",\n    \"scasd\": \"STRING_MEMORY\",\n\n    # INTERRUPT/SYSCALL\n    \"int\": \"INTERRUPT_SYSCALL\",\n    \"syscall\": \"INTERRUPT_SYSCALL\",\n    \"sysenter\": \"INTERRUPT_SYSCALL\",\n}\n\ndef group_tokens(tokens):\n    return [opcode_to_group.get(tok, \"OTHER\") for tok in tokens]\n\navailable_df[\"group_tokens\"] = available_df[\"raw_tokens\"].apply(group_tokens)\n\n# -----------------------------\n# 9. Basic per-sample feature summary\n# -----------------------------\ndef safe_ratio(count, total):\n    return float(count) / total if total else 0.0\n\nsummary_rows = []\nfor _, row in available_df.iterrows():\n    raw_tokens = row[\"raw_tokens\"]\n    group_tokens = row[\"group_tokens\"]\n    c = Counter(group_tokens)\n    total = len(group_tokens)\n\n    summary_rows.append({\n        \"Id\": row[\"Id\"],\n        \"Class\": row[\"Class\"],\n        \"opcode_count\": len(raw_tokens),\n        \"unique_opcode_count\": len(set(raw_tokens)),\n        \"group_count\": len(group_tokens),\n        \"unique_group_count\": len(set(group_tokens)),\n        \"move_count\": c[\"MOVE\"],\n        \"arith_count\": c[\"ARITH\"],\n        \"logic_count\": c[\"LOGIC\"],\n        \"compare_count\": c[\"COMPARE\"],\n        \"jump_count\": c[\"JUMP\"],\n        \"call_count\": c[\"CALL\"],\n        \"stack_count\": c[\"STACK\"],\n        \"string_memory_count\": c[\"STRING_MEMORY\"],\n        \"interrupt_syscall_count\": c[\"INTERRUPT_SYSCALL\"],\n        \"other_count\": c[\"OTHER\"],\n        \"move_ratio\": safe_ratio(c[\"MOVE\"], total),\n        \"jump_ratio\": safe_ratio(c[\"JUMP\"], total),\n        \"call_ratio\": safe_ratio(c[\"CALL\"], total),\n        \"other_ratio\": safe_ratio(c[\"OTHER\"], total),\n    })\n\nfeature_df = pd.DataFrame(summary_rows)\n\n# -----------------------------\n# 10. NSF-style tokenization\n# -----------------------------\n# Treat each sample as a document of tokens, then mean-pool Word2Vec vectors.\n\ndocuments_raw = available_df[\"raw_tokens\"].tolist()\ndocuments_group = available_df[\"group_tokens\"].tolist()\n\ndef document_vector(model, tokens):\n    kept = [t for t in tokens if t in model.wv.key_to_index]\n    if not kept:\n        return np.zeros(model.vector_size, dtype=np.float32)\n    return np.mean(model.wv[kept], axis=0)\n\nw2v_raw = Word2Vec(\n    sentences=documents_raw,\n    vector_size=VECTOR_SIZE,\n    window=WINDOW_SIZE,\n    min_count=MIN_COUNT,\n    workers=4,\n    seed=RANDOM_SEED\n)\n\nw2v_group = Word2Vec(\n    sentences=documents_group,\n    vector_size=VECTOR_SIZE,\n    window=WINDOW_SIZE,\n    min_count=MIN_COUNT,\n    workers=4,\n    seed=RANDOM_SEED\n)\n\nraw_vectors = np.array([document_vector(w2v_raw, doc) for doc in documents_raw])\ngroup_vectors = np.array([document_vector(w2v_group, doc) for doc in documents_group])\n\nnsf_raw_df = pd.DataFrame(raw_vectors)\nnsf_raw_df.insert(0, \"Id\", available_df[\"Id\"].values)\nnsf_raw_df.insert(1, \"Class\", available_df[\"Class\"].values)\n\nnsf_group_df = pd.DataFrame(group_vectors)\nnsf_group_df.insert(0, \"Id\", available_df[\"Id\"].values)\nnsf_group_df.insert(1, \"Class\", available_df[\"Class\"].values)\n\n# -----------------------------\n# 11. Tokenization-Testing-style tokenization\n# -----------------------------\ndef make_unigrams(tokens):\n    return list(tokens)\n\ndef make_bigrams(tokens):\n    if len(tokens) < 2:\n        return []\n    return [f\"{tokens[i]}_{tokens[i+1]}\" for i in range(len(tokens) - 1)]\n\navailable_df[\"unigrams_raw\"] = available_df[\"raw_tokens\"].apply(make_unigrams)\navailable_df[\"bigrams_raw\"] = available_df[\"raw_tokens\"].apply(make_bigrams)\n\navailable_df[\"unigrams_group\"] = available_df[\"group_tokens\"].apply(make_unigrams)\navailable_df[\"bigrams_group\"] = available_df[\"group_tokens\"].apply(make_bigrams)\n\ndef top_vocab(token_lists, top_n):\n    counter = Counter()\n    for row in token_lists:\n        counter.update(row)\n    return set(tok for tok, _ in counter.most_common(top_n))\n\ndef filter_vocab(tokens, vocab):\n    return [t for t in tokens if t in vocab]\n\nraw_uni_vocab = top_vocab(available_df[\"unigrams_raw\"], TOP_UNIGRAMS)\nraw_bi_vocab = top_vocab(available_df[\"bigrams_raw\"], TOP_BIGRAMS)\n\ngroup_uni_vocab = top_vocab(available_df[\"unigrams_group\"], TOP_UNIGRAMS)\ngroup_bi_vocab = top_vocab(available_df[\"bigrams_group\"], TOP_BIGRAMS)\n\navailable_df[\"unigrams_raw_top\"] = available_df[\"unigrams_raw\"].apply(lambda x: filter_vocab(x, raw_uni_vocab))\navailable_df[\"bigrams_raw_top\"] = available_df[\"bigrams_raw\"].apply(lambda x: filter_vocab(x, raw_bi_vocab))\n\navailable_df[\"unigrams_group_top\"] = available_df[\"unigrams_group\"].apply(lambda x: filter_vocab(x, group_uni_vocab))\navailable_df[\"bigrams_group_top\"] = available_df[\"bigrams_group\"].apply(lambda x: filter_vocab(x, group_bi_vocab))\n\n# -----------------------------\n# 12. Compare tokenization outputs\n# -----------------------------\ncomparison_rows = []\nfor _, row in available_df.iterrows():\n    comparison_rows.append({\n        \"Id\": row[\"Id\"],\n        \"Class\": row[\"Class\"],\n        \"raw_token_count\": len(row[\"raw_tokens\"]),\n        \"group_token_count\": len(row[\"group_tokens\"]),\n        \"raw_unigram_count\": len(row[\"unigrams_raw\"]),\n        \"group_unigram_count\": len(row[\"unigrams_group\"]),\n        \"raw_bigram_count\": len(row[\"bigrams_raw\"]),\n        \"group_bigram_count\": len(row[\"bigrams_group\"]),\n        \"raw_unigram_top_count\": len(row[\"unigrams_raw_top\"]),\n        \"group_unigram_top_count\": len(row[\"unigrams_group_top\"]),\n        \"raw_bigram_top_count\": len(row[\"bigrams_raw_top\"]),\n        \"group_bigram_top_count\": len(row[\"bigrams_group_top\"]),\n        \"nsf_vector_dim_raw\": raw_vectors.shape[1],\n        \"nsf_vector_dim_group\": group_vectors.shape[1],\n    })\n\ncomparison_df = pd.DataFrame(comparison_rows)\n\nsummary_df = pd.DataFrame([{\n    \"num_samples\": len(available_df),\n    \"raw_unigram_vocab_size\": len(set(t for row in available_df[\"unigrams_raw\"] for t in row)),\n    \"group_unigram_vocab_size\": len(set(t for row in available_df[\"unigrams_group\"] for t in row)),\n    \"raw_bigram_vocab_size\": len(set(t for row in available_df[\"bigrams_raw\"] for t in row)),\n    \"group_bigram_vocab_size\": len(set(t for row in available_df[\"bigrams_group\"] for t in row)),\n    \"raw_unigram_top_vocab_size\": len(raw_uni_vocab),\n    \"raw_bigram_top_vocab_size\": len(raw_bi_vocab),\n    \"group_unigram_top_vocab_size\": len(group_uni_vocab),\n    \"group_bigram_top_vocab_size\": len(group_bi_vocab),\n    \"avg_raw_token_count\": float(comparison_df[\"raw_token_count\"].mean()),\n    \"avg_group_token_count\": float(comparison_df[\"group_token_count\"].mean()),\n    \"avg_raw_bigram_count\": float(comparison_df[\"raw_bigram_count\"].mean()),\n    \"avg_group_bigram_count\": float(comparison_df[\"group_bigram_count\"].mean()),\n    \"nsf_vector_dim\": VECTOR_SIZE\n}])\n\n# -----------------------------\n# 13. Save outputs\n# -----------------------------\navailable_export = available_df[[\"Id\", \"Class\"]].copy()\navailable_export[\"raw_tokens\"] = available_df[\"raw_tokens\"].apply(lambda x: \" \".join(x))\navailable_export[\"group_tokens\"] = available_df[\"group_tokens\"].apply(lambda x: \" \".join(x))\navailable_export[\"unigrams_raw\"] = available_df[\"unigrams_raw\"].apply(lambda x: \" \".join(x))\navailable_export[\"bigrams_raw\"] = available_df[\"bigrams_raw\"].apply(lambda x: \" \".join(x))\navailable_export[\"unigrams_group\"] = available_df[\"unigrams_group\"].apply(lambda x: \" \".join(x))\navailable_export[\"bigrams_group\"] = available_df[\"bigrams_group\"].apply(lambda x: \" \".join(x))\navailable_export[\"unigrams_raw_top\"] = available_df[\"unigrams_raw_top\"].apply(lambda x: \" \".join(x))\navailable_export[\"bigrams_raw_top\"] = available_df[\"bigrams_raw_top\"].apply(lambda x: \" \".join(x))\navailable_export[\"unigrams_group_top\"] = available_df[\"unigrams_group_top\"].apply(lambda x: \" \".join(x))\navailable_export[\"bigrams_group_top\"] = available_df[\"bigrams_group_top\"].apply(lambda x: \" \".join(x))\n\navailable_export.to_csv(os.path.join(WORK_DIR, \"sampled_opcode_sequences.csv\"), index=False)\nfeature_df.to_csv(os.path.join(WORK_DIR, \"opcode_group_feature_summary.csv\"), index=False)\nnsf_raw_df.to_csv(os.path.join(WORK_DIR, \"nsf_raw_vectors.csv\"), index=False)\nnsf_group_df.to_csv(os.path.join(WORK_DIR, \"nsf_group_vectors.csv\"), index=False)\ncomparison_df.to_csv(os.path.join(WORK_DIR, \"tokenization_comparison.csv\"), index=False)\nsummary_df.to_csv(os.path.join(WORK_DIR, \"tokenization_summary.csv\"), index=False)\navailable_export.to_pickle(os.path.join(WORK_DIR, \"tokenization_outputs.pkl\"))\n\n# -----------------------------\n# 14. Preview outputs\n# -----------------------------\nprint(\"Sampled malware subset:\")\ndisplay(available_df[[\"Id\", \"Class\", \"asm_path\"]].head())\n\nprint(\"\\nOpcode sequence examples:\")\npreview = available_df[[\"Id\", \"Class\", \"raw_tokens\", \"group_tokens\"]].head().copy()\npreview[\"raw_tokens\"] = preview[\"raw_tokens\"].apply(lambda x: \" \".join(x[:40]))\npreview[\"group_tokens\"] = preview[\"group_tokens\"].apply(lambda x: \" \".join(x[:40]))\ndisplay(preview)\n\nprint(\"\\nNSF raw vector preview:\")\ndisplay(nsf_raw_df.head())\n\nprint(\"\\nFeature summary preview:\")\ndisplay(feature_df.head())\n\nprint(\"\\nTokenization comparison preview:\")\ndisplay(comparison_df.head())\n\nprint(\"\\nTokenization summary:\")\ndisplay(summary_df)\n\nprint(\"\\nSaved files:\")\nfor f in sorted(os.listdir(WORK_DIR)):\n    if f.endswith(\".csv\") or f.endswith(\".pkl\"):\n        print(\"-\", f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-30T20:47:49.073955Z","iopub.execute_input":"2026-03-30T20:47:49.074498Z","iopub.status.idle":"2026-03-30T20:52:52.180319Z","shell.execute_reply.started":"2026-03-30T20:47:49.074463Z","shell.execute_reply":"2026-03-30T20:52:52.179305Z"}},"outputs":[],"execution_count":null}]}