{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":4117,"databundleVersionId":46665,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# =========================\n# PATHS\n# =========================\nINPUT_DIR = \"/kaggle/input/malware-classification\"\nARCHIVE_PATH = os.path.join(INPUT_DIR, \"train.7z\")\nLABEL_CSV = \"/kaggle/input/malware-classification/trainLabels.csv\"\nOUTPUT_DIR = \"/kaggle/working/asm_labeled\"\n\n# =========================\n# LABEL → FAMILY MAP\n# =========================\nlabel_to_family = {\n    1: \"Ramnit\",\n    2: \"Lollipop\",\n    3: \"Kelihos_ver3\",\n    8: \"Obfuscator.ACY\",\n    9: \"Gatak\"\n}\n\n# Create folders\nos.makedirs(OUTPUT_DIR, exist_ok=True)\nfor family in label_to_family.values():\n    os.makedirs(os.path.join(OUTPUT_DIR, family), exist_ok=True)\n\nprint(LABEL_CSV)\nprint(\"✅ Output folders ready\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T14:37:43.280411Z","iopub.execute_input":"2025-12-13T14:37:43.280702Z","iopub.status.idle":"2025-12-13T14:37:43.287933Z","shell.execute_reply.started":"2025-12-13T14:37:43.280675Z","shell.execute_reply":"2025-12-13T14:37:43.287122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read CSV\ndf = pd.read_csv(LABEL_CSV)\n\n# Keep only selected classes\ndf = df[df[\"Class\"].isin(label_to_family.keys())]\n\nprint(f\"📊 Total ASM files to extract: {len(df)}\")\ndf.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T14:37:49.459883Z","iopub.execute_input":"2025-12-13T14:37:49.460150Z","iopub.status.idle":"2025-12-13T14:37:49.567099Z","shell.execute_reply.started":"2025-12-13T14:37:49.460128Z","shell.execute_reply":"2025-12-13T14:37:49.566266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import subprocess\nfrom tqdm import tqdm\n\nmissing = 0\nextracted = 0\n\nfor _, row in tqdm(df.iterrows(), total=len(df), desc=\"Extracting ASM files\"):\n    file_id = row[\"Id\"]\n    family = label_to_family[row[\"Class\"]]\n\n    asm_name = f\"{file_id}.asm\"\n    output_path = os.path.join(OUTPUT_DIR, family)\n\n    cmd = [\n        \"7z\", \"x\",\n        ARCHIVE_PATH,\n        asm_name,\n        f\"-o{output_path}\",\n        \"-y\"\n    ]\n\n    result = subprocess.run(\n        cmd,\n        stdout=subprocess.DEVNULL,\n        stderr=subprocess.DEVNULL\n    )\n\n    if result.returncode == 0:\n        extracted += 1\n    else:\n        missing += 1\n\nprint(\"\\n✅ Extraction complete\")\nprint(f\"✔ Extracted files : {extracted}\")\nprint(f\"⚠ Missing files  : {missing}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T14:37:56.601801Z","iopub.execute_input":"2025-12-13T14:37:56.602060Z","iopub.status.idle":"2025-12-13T14:45:09.458245Z","shell.execute_reply.started":"2025-12-13T14:37:56.602040Z","shell.execute_reply":"2025-12-13T14:45:09.457620Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls /kaggle/input/malware-classification","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T14:37:34.328462Z","iopub.execute_input":"2025-12-13T14:37:34.329345Z","iopub.status.idle":"2025-12-13T14:37:34.453970Z","shell.execute_reply.started":"2025-12-13T14:37:34.329274Z","shell.execute_reply":"2025-12-13T14:37:34.453339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!find /kaggle/working/asm_labeled -name \"*.asm\" | wc -l","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T15:01:57.268390Z","iopub.execute_input":"2025-12-13T15:01:57.269177Z","iopub.status.idle":"2025-12-13T15:01:57.392914Z","shell.execute_reply.started":"2025-12-13T15:01:57.269139Z","shell.execute_reply":"2025-12-13T15:01:57.392208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport subprocess\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# =========================\n# PATHS\n# =========================\nINPUT_DIR = \"/kaggle/input/malware-classification\"\nARCHIVE_PATH = os.path.join(INPUT_DIR, \"train.7z\")\nLABEL_CSV = os.path.join(INPUT_DIR, \"trainLabels.csv\")\nOUTPUT_DIR = \"/kaggle/working/asm_labeled\"\n\n# =========================\n# LABEL MAP\n# =========================\nlabel_to_family = {\n    9: \"Gatak\"\n}\n\n# =========================\n# PREPARE FOLDERS\n# =========================\nfor fam in label_to_family.values():\n    os.makedirs(os.path.join(OUTPUT_DIR, fam), exist_ok=True)\n\n# =========================\n# READ CSV\n# =========================\ndf = pd.read_csv(LABEL_CSV)\ndf = df[df[\"Class\"].isin(label_to_family.keys())]\n\nprint(f\"\\n📊 CSV selected samples: {len(df)}\")\n\n# =========================\n# TRACE STRUCTURES\n# =========================\nexpected_ids = set(df[\"Id\"])\nextracted_ids = set()\nfamily_expected = defaultdict(int)\nfamily_extracted = defaultdict(int)\nfailed_ids = []\n\n# =========================\n# EXTRACTION\n# =========================\nfor _, row in tqdm(df.iterrows(), total=len(df), desc=\"Extracting ASM with trace\"):\n    file_id = row[\"Id\"]\n    family = label_to_family[row[\"Class\"]]\n    family_expected[family] += 1\n\n    asm_in_7z = f\"train/{file_id}.asm\"\n    out_dir = os.path.join(OUTPUT_DIR, family)\n\n    cmd = [\n        \"7z\", \"x\",\n        ARCHIVE_PATH,\n        asm_in_7z,\n        f\"-o{out_dir}\",\n        \"-y\"\n    ]\n\n    result = subprocess.run(\n        cmd,\n        stdout=subprocess.DEVNULL,\n        stderr=subprocess.DEVNULL\n    )\n\n    if result.returncode == 0:\n        extracted_ids.add(file_id)\n        family_extracted[family] += 1\n    else:\n        failed_ids.append(file_id)\n\nprint(\"\\n✅ Extraction phase finished\")\nprint(f\"Extracted files per family: {dict(family_extracted)}\")\nprint(f\"Failed extractions: {len(failed_ids)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T17:51:34.397508Z","iopub.execute_input":"2025-12-13T17:51:34.397836Z","iopub.status.idle":"2025-12-13T18:04:07.822595Z","shell.execute_reply.started":"2025-12-13T17:51:34.397815Z","shell.execute_reply":"2025-12-13T18:04:07.821668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ndisk_ids = set()\n\nfor root, _, files in os.walk(OUTPUT_DIR):\n    for f in files:\n        if f.endswith(\".asm\"):\n            disk_ids.add(f.replace(\".asm\", \"\"))\n\nprint(f\"\\n📁 ASM files found on disk: {len(disk_ids)}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_on_disk = expected_ids - disk_ids\nextra_on_disk = disk_ids - expected_ids\n\nprint(\"\\n🔍 CONSISTENCY REPORT\")\nprint(f\"✔ Expected from CSV     : {len(expected_ids)}\")\nprint(f\"✔ Extracted (7z success): {len(extracted_ids)}\")\nprint(f\"✔ Found on disk         : {len(disk_ids)}\")\nprint(f\"❌ Missing on disk      : {len(missing_on_disk)}\")\nprint(f\"⚠ Extra on disk         : {len(extra_on_disk)}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n📂 FAMILY-WISE TRACE\")\nprint(\"-\" * 45)\n\nfor fam in label_to_family.values():\n    print(\n        f\"{fam:15s} | \"\n        f\"Expected: {family_expected[fam]:5d} | \"\n        f\"Extracted: {family_extracted[fam]:5d}\"\n    )\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls /kaggle/working/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T15:19:48.061337Z","iopub.execute_input":"2025-12-13T15:19:48.062013Z","iopub.status.idle":"2025-12-13T15:19:48.181084Z","shell.execute_reply.started":"2025-12-13T15:19:48.061958Z","shell.execute_reply":"2025-12-13T15:19:48.179786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport subprocess\nfrom collections import defaultdict\nfrom multiprocessing import Pool, cpu_count\nimport math\n\n# =========================\n# PATHS\n# =========================\nINPUT_DIR = \"/kaggle/input/malware-classification\"\nARCHIVE_PATH = os.path.join(INPUT_DIR, \"train.7z\")\nLABEL_CSV = os.path.join(INPUT_DIR, \"trainLabels.csv\")\nOUTPUT_DIR = \"/kaggle/working/asm_labeled\"\n\n# =========================\n# LABEL MAP\n# =========================\nlabel_to_family = {\n    9: \"Gatak\"\n}\n\n# =========================\n# PREPARE FOLDERS\n# =========================\nfor fam in label_to_family.values():\n    os.makedirs(os.path.join(OUTPUT_DIR, fam), exist_ok=True)\n\n# =========================\n# READ CSV\n# =========================\ndf = pd.read_csv(LABEL_CSV)\ndf = df[df[\"Class\"].isin(label_to_family.keys())]\nprint(f\"📊 CSV selected samples: {len(df)}\")\n\n# =========================\n# CHUNKING\n# =========================\nnum_chunks = 3\nchunk_size = math.ceil(len(df) / num_chunks)\nchunks = [df[i*chunk_size : (i+1)*chunk_size] for i in range(num_chunks)]\n\n# =========================\n# EXTRACTION FUNCTION\n# =========================\ndef extract_files(df_chunk):\n    extracted_ids = set()\n    family_extracted = defaultdict(int)\n    failed_ids = []\n\n    for _, row in df_chunk.iterrows():\n        file_id = row[\"Id\"]\n        family = label_to_family[row[\"Class\"]]\n        out_dir = os.path.join(OUTPUT_DIR, family)\n        asm_in_7z = f\"train/{file_id}.asm\"\n\n        cmd = [\n            \"7z\", \"x\",\n            ARCHIVE_PATH,\n            asm_in_7z,\n            f\"-o{out_dir}\",\n            \"-y\"\n        ]\n\n        result = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n        if result.returncode == 0:\n            extracted_ids.add(file_id)\n            family_extracted[family] += 1\n        else:\n            failed_ids.append(file_id)\n\n    return extracted_ids, dict(family_extracted), failed_ids\n\n# =========================\n# PARALLEL EXTRACTION\n# =========================\nif __name__ == \"__main__\":\n    with Pool(min(num_chunks, cpu_count())) as pool:\n        results = pool.map(extract_files, chunks)\n\n    all_extracted_ids = set()\n    all_family_extracted = defaultdict(int)\n    all_failed_ids = []\n\n    for extracted_ids, family_extracted, failed_ids in results:\n        all_extracted_ids.update(extracted_ids)\n        for fam, count in family_extracted.items():\n            all_family_extracted[fam] += count\n        all_failed_ids.extend(failed_ids)\n\n    print(\"\\n✅ All chunks finished\")\n    print(f\"Total extracted files per family: {dict(all_family_extracted)}\")\n    print(f\"Total failed extractions: {len(all_failed_ids)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T18:08:34.462171Z","iopub.execute_input":"2025-12-13T18:08:34.462411Z","iopub.status.idle":"2025-12-13T18:56:55.687708Z","shell.execute_reply.started":"2025-12-13T18:08:34.462383Z","shell.execute_reply":"2025-12-13T18:56:55.686920Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!7z a -tzip /kaggle/working/asm_labeled.zip /kaggle/working/asm_labeled/*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T19:02:09.297773Z","iopub.execute_input":"2025-12-13T19:02:09.298497Z","iopub.status.idle":"2025-12-13T19:12:31.638799Z","shell.execute_reply.started":"2025-12-13T19:02:09.298460Z","shell.execute_reply":"2025-12-13T19:12:31.637937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import HTML\nimport os\n\nzip_file = '/kaggle/working/asm_labeled.zip'\nfilename = os.path.basename(zip_file)\nsize_mb = os.path.getsize(zip_file) / (1024 * 1024)\n\nHTML(f'''\n<div style=\"padding: 20px; background: #f5f5f5; border-radius: 10px;\">\n    <h3>📁 File Ready for Download</h3>\n    <p><strong>File:</strong> {filename}</p>\n    <p><strong>Size:</strong> {size_mb:.2f} MB</p>\n    <p><a href=\"{zip_file}\" download style=\"\n        display: inline-block;\n        padding: 10px 20px;\n        background: #0066cc;\n        color: white;\n        text-decoration: none;\n        border-radius: 5px;\n        font-weight: bold;\n    \">⬇️ Download {filename}</a></p>\n</div>\n''')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T19:14:40.129864Z","iopub.execute_input":"2025-12-13T19:14:40.130523Z","iopub.status.idle":"2025-12-13T19:14:40.137167Z","shell.execute_reply.started":"2025-12-13T19:14:40.130501Z","shell.execute_reply":"2025-12-13T19:14:40.136535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -lh /kaggle/working\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T19:28:55.728083Z","iopub.execute_input":"2025-12-13T19:28:55.728355Z","iopub.status.idle":"2025-12-13T19:28:55.899496Z","shell.execute_reply.started":"2025-12-13T19:28:55.728334Z","shell.execute_reply":"2025-12-13T19:28:55.898861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!7z a -tzip -v1g /kaggle/working/asm_labeled_parts.zip /kaggle/working/asm_labeled/*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T19:31:57.377397Z","iopub.execute_input":"2025-12-13T19:31:57.377695Z","iopub.status.idle":"2025-12-13T19:42:18.862897Z","shell.execute_reply.started":"2025-12-13T19:31:57.377669Z","shell.execute_reply":"2025-12-13T19:42:18.862214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -lh /kaggle/working\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T19:46:22.719870Z","iopub.execute_input":"2025-12-13T19:46:22.720771Z","iopub.status.idle":"2025-12-13T19:46:22.869149Z","shell.execute_reply.started":"2025-12-13T19:46:22.720727Z","shell.execute_reply":"2025-12-13T19:46:22.868193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink('/kaggle/working/asm_labeled_parts.zip.001')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T19:46:51.217543Z","iopub.execute_input":"2025-12-13T19:46:51.218336Z","iopub.status.idle":"2025-12-13T19:46:51.223459Z","shell.execute_reply.started":"2025-12-13T19:46:51.218308Z","shell.execute_reply":"2025-12-13T19:46:51.222904Z"}},"outputs":[],"execution_count":null}]}