{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"databundleVersionId":46665,"isSourceIdPinned":false,"sourceId":4117,"sourceType":"competition"}],"dockerImageVersionId":31328,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"papermill":{"default_parameters":{},"duration":23296.896951,"end_time":"2026-04-08T17:31:49.379092+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-04-08T11:03:32.482141+00:00","version":"2.7.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"295ab83fb95c4761a089aa74d602d571":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3261ae7e7a114b208028554319262106":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"40a23981fbb244608dea9e587b4548c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_295ab83fb95c4761a089aa74d602d571","placeholder":"​","style":"IPY_MODEL_4e851c4774be4019b141ad76855f828a","tabbable":null,"tooltip":null,"value":" 28/28 [6:28:09<00:00, 725.96s/it]"}},"4e851c4774be4019b141ad76855f828a":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"60660ea7c6574158a414fedd55a303a6":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_d5f0ede8f923407aa9dbafcab8f94363","placeholder":"​","style":"IPY_MODEL_8cbdca3fd9ac4a2e842a0e563ead31a1","tabbable":null,"tooltip":null,"value":"Processing Batches: 100%"}},"8cbdca3fd9ac4a2e842a0e563ead31a1":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"96a65453d0df48d4ad7c7920ee619eb4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_60660ea7c6574158a414fedd55a303a6","IPY_MODEL_be9336e1719740f3961aec9ee9615cc9","IPY_MODEL_40a23981fbb244608dea9e587b4548c4"],"layout":"IPY_MODEL_9e67b79f6ed146e7b9093fb7574ec348","tabbable":null,"tooltip":null}},"9e67b79f6ed146e7b9093fb7574ec348":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"be9336e1719740f3961aec9ee9615cc9":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_de60676d18334207b05a5b79932de14f","max":28,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3261ae7e7a114b208028554319262106","tabbable":null,"tooltip":null,"value":28}},"d5f0ede8f923407aa9dbafcab8f94363":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"de60676d18334207b05a5b79932de14f":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"2a17bb63","cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport subprocess\nimport math\nimport zlib\nimport zipfile \nfrom tqdm.notebook import tqdm\n\n# ==========================================\n# CẤU HÌNH SONG SONG (ĐIỀU CHỈNH Ở MỖI KERNEL)\n# ==========================================\nSTART_INDEX = 9500      \nEND_INDEX = 11000   \nBATCH_SIZE = 50    \n\n# ĐƯỜNG DẪN DỮ LIỆU TEST\nDATA_DIR = '/kaggle/input/competitions/malware-classification'\nTEST_ZIP = os.path.join(DATA_DIR, 'test.7z')\nSAMPLE_SUB_CSV = os.path.join(DATA_DIR, 'sampleSubmission.csv') # Dùng file này để lấy ID test\n\nWORKING_DIR = '/kaggle/working'\nTEMP_DIR = os.path.join(WORKING_DIR, 'temp_unzip')\nOUT_BYTES_DIR = os.path.join(WORKING_DIR, 'test_images_bytes_temp') \nOUT_ASM_DIR = os.path.join(WORKING_DIR, 'test_images_asm_temp')\n\n# Tên 2 file ZIP tổng hợp cuối cùng dành cho tập Test\nFINAL_BYTES_ZIP = os.path.join(WORKING_DIR, f'test_images_bytes_{START_INDEX}_to_{END_INDEX}.zip')\nFINAL_ASM_ZIP = os.path.join(WORKING_DIR, f'test_images_asm_{START_INDEX}_to_{END_INDEX}.zip')\n\n# --- FIX BẢO MẬT DỮ LIỆU: Xóa file ZIP cũ nếu chạy lại Kernel ---\nfor file_path in [FINAL_BYTES_ZIP, FINAL_ASM_ZIP]:\n    if os.path.exists(file_path):\n        os.remove(file_path)\n\n# Khởi tạo thư mục tạm\nfor d in [TEMP_DIR, OUT_BYTES_DIR, OUT_ASM_DIR]:\n    os.makedirs(d, exist_ok=True)\n\nIMAGE_SIZE = (256, 256)\nWINDOW_SIZE = 32\n\nprint(f\"BẮT ĐẦU: Tạo 2 file zip TEST từ Index {START_INDEX} đến {END_INDEX}...\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2026-04-08T11:03:35.821808Z","iopub.status.busy":"2026-04-08T11:03:35.821442Z","iopub.status.idle":"2026-04-08T11:03:37.677003Z","shell.execute_reply":"2026-04-08T11:03:37.675774Z"},"papermill":{"duration":1.861877,"end_time":"2026-04-08T11:03:37.67922+00:00","exception":false,"start_time":"2026-04-08T11:03:35.817343+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"86a6e58e","cell_type":"code","source":"# .bytes\ndef process_bytes_rgb(bytes_path, out_path):\n    if os.path.exists(out_path): return\n    byte_list = []\n    \n    try:\n        with open(bytes_path, 'r') as f:\n            for line in f:\n                tokens = line.split()\n                for token in tokens[1:]:\n                    if token == '??':\n                        byte_list.append(0)\n                    else:\n                        byte_list.append(int(token, 16))\n    except Exception: return\n\n    if not byte_list: return\n    byte_array = np.array(byte_list, dtype=np.uint8)\n    \n    # Kênh Đỏ (R)\n    R = byte_array\n    \n    # Chia block để tính đặc trưng\n    pad_len = math.ceil(len(byte_array) / WINDOW_SIZE) * WINDOW_SIZE - len(byte_array)\n    padded_array = np.pad(byte_array, (0, pad_len), 'constant', constant_values=0)\n    blocks = padded_array.reshape(-1, WINDOW_SIZE)\n    \n    # Kênh Xanh Lá (G)\n    entropy_list = []\n    for block in blocks:\n        _, counts = np.unique(block, return_counts=True)\n        probs = counts / WINDOW_SIZE\n        ent = -np.sum(probs * np.log2(probs + 1e-10))\n        entropy_list.append(min(255, int((ent / 8.0) * 255)))\n    G = np.repeat(entropy_list, WINDOW_SIZE)[:len(byte_array)].astype(np.uint8)\n    \n    # Kênh Xanh Dương (B)\n    stds = np.std(blocks, axis=1)\n    stds_norm = np.clip((stds / 128.0) * 255, 0, 255)\n    B = np.repeat(stds_norm, WINDOW_SIZE)[:len(byte_array)].astype(np.uint8)\n    \n    # Dựng ảnh vuông\n    width = int(math.ceil(math.sqrt(len(byte_array))))\n    pad_size = (width * width) - len(byte_array)\n    \n    R_pad = np.pad(R, (0, pad_size), 'constant').reshape(width, width)\n    G_pad = np.pad(G, (0, pad_size), 'constant').reshape(width, width)\n    B_pad = np.pad(B, (0, pad_size), 'constant').reshape(width, width)\n    \n    img_bgr = np.dstack((B_pad, G_pad, R_pad))\n    final_img = cv2.resize(img_bgr, IMAGE_SIZE, interpolation=cv2.INTER_AREA)\n    cv2.imwrite(out_path, final_img)","metadata":{"execution":{"iopub.execute_input":"2026-04-08T11:03:37.684537Z","iopub.status.busy":"2026-04-08T11:03:37.684119Z","iopub.status.idle":"2026-04-08T11:03:37.69529Z","shell.execute_reply":"2026-04-08T11:03:37.694489Z"},"papermill":{"duration":0.016006,"end_time":"2026-04-08T11:03:37.697161+00:00","exception":false,"start_time":"2026-04-08T11:03:37.681155+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"9a641838","cell_type":"code","source":"# .asm\ndef process_asm_markov(asm_path, out_path):\n    if os.path.exists(out_path): return\n    transition_matrix = np.zeros((256, 256), dtype=np.float32)\n    prev_idx = None\n    \n    try:\n        with open(asm_path, 'r', encoding='latin1', errors='ignore') as f:\n            for line in f:\n                parts = line.split()\n                if len(parts) > 1:\n                    opcode = parts[1].lower()\n                    if not opcode.isalnum(): continue\n                    \n                    # Băm Opcode thành tọa độ pixel\n                    curr_idx = zlib.crc32(opcode.encode('utf-8')) % 256\n                    \n                    if prev_idx is not None:\n                        transition_matrix[prev_idx, curr_idx] += 1\n                    prev_idx = curr_idx\n    except Exception: return\n\n    # Chuẩn hóa theo thang Logarit\n    matrix_log = np.log1p(transition_matrix)\n    max_val = np.max(matrix_log)\n    \n    if max_val > 0:\n        img_markov = (matrix_log / max_val) * 255.0\n    else:\n        img_markov = matrix_log\n        \n    cv2.imwrite(out_path, img_markov.astype(np.uint8))","metadata":{"execution":{"iopub.execute_input":"2026-04-08T11:03:37.702044Z","iopub.status.busy":"2026-04-08T11:03:37.70168Z","iopub.status.idle":"2026-04-08T11:03:37.709291Z","shell.execute_reply":"2026-04-08T11:03:37.708349Z"},"papermill":{"duration":0.012117,"end_time":"2026-04-08T11:03:37.711053+00:00","exception":false,"start_time":"2026-04-08T11:03:37.698936+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"c424e6c5","cell_type":"code","source":"def save_checkpoint_and_clear():\n    \"\"\"Ghi nối (append) các ảnh PNG mới vào 2 file ZIP tổng và xóa ảnh thô\"\"\"\n    \n    # 1. Xử lý thư mục Bytes\n    if len(os.listdir(OUT_BYTES_DIR)) > 0:\n        with zipfile.ZipFile(FINAL_BYTES_ZIP, 'a', zipfile.ZIP_DEFLATED) as zipf:\n            for filename in os.listdir(OUT_BYTES_DIR):\n                filepath = os.path.join(OUT_BYTES_DIR, filename)\n                zipf.write(filepath, arcname=filename) \n                os.remove(filepath)\n                \n    # 2. Xử lý thư mục ASM\n    if len(os.listdir(OUT_ASM_DIR)) > 0:\n        with zipfile.ZipFile(FINAL_ASM_ZIP, 'a', zipfile.ZIP_DEFLATED) as zipf:\n            for filename in os.listdir(OUT_ASM_DIR):\n                filepath = os.path.join(OUT_ASM_DIR, filename)\n                zipf.write(filepath, arcname=filename)\n                os.remove(filepath)\n\ndef run_pipeline():\n    # Đọc ID từ sampleSubmission.csv thay vì trainLabels.csv\n    df_sub = pd.read_csv(SAMPLE_SUB_CSV)\n    all_ids = df_sub['Id'].tolist()[START_INDEX:END_INDEX]\n    total_files = len(all_ids)\n    \n    for i in tqdm(range(0, total_files, BATCH_SIZE), desc=\"Processing Batches\"):\n        batch_ids = all_ids[i:i+BATCH_SIZE]\n        \n        current_batch_end = START_INDEX + i + len(batch_ids)\n        \n        list_file_path = os.path.join(WORKING_DIR, 'batch_list.txt')\n        with open(list_file_path, 'w') as f:\n            for m_id in batch_ids:\n                f.write(f\"{m_id}.bytes\\n\")\n                f.write(f\"{m_id}.asm\\n\")\n                \n        # GIẢI NÉN TỪ TEST_ZIP\n        cmd = f\"7z e {TEST_ZIP} -o{TEMP_DIR} -ir@{list_file_path} -y\"\n        subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)\n        \n        for m_id in batch_ids:\n            bytes_in = os.path.join(TEMP_DIR, f\"{m_id}.bytes\")\n            asm_in = os.path.join(TEMP_DIR, f\"{m_id}.asm\")\n            \n            bytes_out = os.path.join(OUT_BYTES_DIR, f\"{m_id}.png\")\n            asm_out = os.path.join(OUT_ASM_DIR, f\"{m_id}.png\")\n            \n            if os.path.exists(bytes_in):\n                process_bytes_rgb(bytes_in, bytes_out)\n                os.remove(bytes_in) \n                \n            if os.path.exists(asm_in):\n                process_asm_markov(asm_in, asm_out)\n                os.remove(asm_in) \n        \n        save_checkpoint_and_clear()\n        gc.collect()\n        \n        tqdm.write(f\"✓ Đã xử lý và nén xong: {current_batch_end}/{total_files + START_INDEX} ảnh\")","metadata":{"execution":{"iopub.execute_input":"2026-04-08T11:03:37.716495Z","iopub.status.busy":"2026-04-08T11:03:37.71582Z","iopub.status.idle":"2026-04-08T11:03:37.726821Z","shell.execute_reply":"2026-04-08T11:03:37.725974Z"},"papermill":{"duration":0.015751,"end_time":"2026-04-08T11:03:37.728576+00:00","exception":false,"start_time":"2026-04-08T11:03:37.712825+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"a51eadf1","cell_type":"code","source":"if __name__ == '__main__':\n    run_pipeline()\n    print(f\"\\nHOÀN TẤT: Toàn bộ dữ liệu tập TEST cụm [{START_INDEX} - {END_INDEX}] đã được gộp gọn!\")\n    \n    # Dọn sạch rác\n    !rm -rf {TEMP_DIR} {OUT_BYTES_DIR} {OUT_ASM_DIR} batch_list.txt","metadata":{"execution":{"iopub.execute_input":"2026-04-08T11:03:37.733722Z","iopub.status.busy":"2026-04-08T11:03:37.733374Z","iopub.status.idle":"2026-04-08T17:31:47.936361Z","shell.execute_reply":"2026-04-08T17:31:47.935109Z"},"papermill":{"duration":23290.210601,"end_time":"2026-04-08T17:31:47.940992+00:00","exception":false,"start_time":"2026-04-08T11:03:37.730391+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}