{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"databundleVersionId":46665,"isSourceIdPinned":false,"sourceId":4117,"sourceType":"competition"}],"dockerImageVersionId":31328,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"papermill":{"default_parameters":{},"duration":23489.425683,"end_time":"2026-04-08T17:20:47.908215+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-04-08T10:49:18.482532+00:00","version":"2.7.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"1ff7a48de0b045639eb24f454fc8d3c8":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"262faebcb15f4a92aff44eb577591806":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"3297375f2b89451fa8d642afcc032bc6":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_58a100c9e6f445289ae4104c589a1560","IPY_MODEL_a4db6340d50148d8ba9edb2d77051726","IPY_MODEL_e40f59a69710453d95c0303cca432e51"],"layout":"IPY_MODEL_bf68a1469c77405c9ec08e5c9a319719","tabbable":null,"tooltip":null}},"4061fb41eec84888ab3e1d466dfed52c":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58a100c9e6f445289ae4104c589a1560":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_1ff7a48de0b045639eb24f454fc8d3c8","placeholder":"​","style":"IPY_MODEL_d3a5279aea2e495cb4258fb32c6c1645","tabbable":null,"tooltip":null,"value":"Processing Batches: 100%"}},"a4db6340d50148d8ba9edb2d77051726":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_aaa2e15e0a0e4270bbf51d5506da7229","max":118,"min":0,"orientation":"horizontal","style":"IPY_MODEL_262faebcb15f4a92aff44eb577591806","tabbable":null,"tooltip":null,"value":118}},"aaa2e15e0a0e4270bbf51d5506da7229":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bf68a1469c77405c9ec08e5c9a319719":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d3a5279aea2e495cb4258fb32c6c1645":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"e40f59a69710453d95c0303cca432e51":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_4061fb41eec84888ab3e1d466dfed52c","placeholder":"​","style":"IPY_MODEL_f13c50dd493d497c9ee0f43e72d88f71","tabbable":null,"tooltip":null,"value":" 118/118 [6:31:20<00:00, 154.64s/it]"}},"f13c50dd493d497c9ee0f43e72d88f71":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"d71b4c60","cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport subprocess\nimport math\nimport zlib\nimport zipfile \nfrom tqdm.notebook import tqdm\n\n# ==========================================\n# CẤU HÌNH SONG SONG (ĐIỀU CHỈNH Ở MỖI KERNEL)\n# ==========================================\nSTART_INDEX = 5000      \nEND_INDEX = 11000   \nBATCH_SIZE = 50    \n\n# ĐƯỜNG DẪN DỮ LIỆU\nDATA_DIR = '/kaggle/input/competitions/malware-classification'\nTRAIN_ZIP = os.path.join(DATA_DIR, 'train.7z')\nLABELS_CSV = os.path.join(DATA_DIR, 'trainLabels.csv')\n\nWORKING_DIR = '/kaggle/working'\nTEMP_DIR = os.path.join(WORKING_DIR, 'temp_unzip')\nOUT_BYTES_DIR = os.path.join(WORKING_DIR, 'images_bytes_temp') \nOUT_ASM_DIR = os.path.join(WORKING_DIR, 'images_asm_temp')\n\n# Tên 2 file ZIP tổng hợp cuối cùng\nFINAL_BYTES_ZIP = os.path.join(WORKING_DIR, f'images_bytes_{START_INDEX}_to_{END_INDEX}.zip')\nFINAL_ASM_ZIP = os.path.join(WORKING_DIR, f'images_asm_{START_INDEX}_to_{END_INDEX}.zip')\n\n# --- FIX BẢO MẬT DỮ LIỆU: Xóa file ZIP cũ nếu chạy lại Kernel ---\nfor file_path in [FINAL_BYTES_ZIP, FINAL_ASM_ZIP]:\n    if os.path.exists(file_path):\n        os.remove(file_path)\n\n# Khởi tạo thư mục tạm\nfor d in [TEMP_DIR, OUT_BYTES_DIR, OUT_ASM_DIR]:\n    os.makedirs(d, exist_ok=True)\n\nIMAGE_SIZE = (256, 256)\nWINDOW_SIZE = 32\n\nprint(f\"BẮT ĐẦU: Tạo 2 file zip tổng từ Index {START_INDEX} đến {END_INDEX}...\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2026-04-08T10:49:22.293111Z","iopub.status.busy":"2026-04-08T10:49:22.292712Z","iopub.status.idle":"2026-04-08T10:49:24.166854Z","shell.execute_reply":"2026-04-08T10:49:24.165526Z"},"papermill":{"duration":1.880534,"end_time":"2026-04-08T10:49:24.168967+00:00","exception":false,"start_time":"2026-04-08T10:49:22.288433+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"376b9238","cell_type":"code","source":"# .bytes\ndef process_bytes_rgb(bytes_path, out_path):\n    if os.path.exists(out_path): return\n    byte_list = []\n    \n    try:\n        with open(bytes_path, 'r') as f:\n            for line in f:\n                tokens = line.split()\n                for token in tokens[1:]:\n                    if token == '??':\n                        byte_list.append(0)\n                    else:\n                        byte_list.append(int(token, 16))\n    except Exception: return\n\n    if not byte_list: return\n    byte_array = np.array(byte_list, dtype=np.uint8)\n    \n    # Kênh Đỏ (R)\n    R = byte_array\n    \n    # Chia block để tính đặc trưng\n    pad_len = math.ceil(len(byte_array) / WINDOW_SIZE) * WINDOW_SIZE - len(byte_array)\n    padded_array = np.pad(byte_array, (0, pad_len), 'constant', constant_values=0)\n    blocks = padded_array.reshape(-1, WINDOW_SIZE)\n    \n    # Kênh Xanh Lá (G)\n    entropy_list = []\n    for block in blocks:\n        _, counts = np.unique(block, return_counts=True)\n        probs = counts / WINDOW_SIZE\n        ent = -np.sum(probs * np.log2(probs + 1e-10))\n        entropy_list.append(min(255, int((ent / 8.0) * 255)))\n    G = np.repeat(entropy_list, WINDOW_SIZE)[:len(byte_array)].astype(np.uint8)\n    \n    # Kênh Xanh Dương (B)\n    stds = np.std(blocks, axis=1)\n    stds_norm = np.clip((stds / 128.0) * 255, 0, 255)\n    B = np.repeat(stds_norm, WINDOW_SIZE)[:len(byte_array)].astype(np.uint8)\n    \n    # Dựng ảnh vuông\n    width = int(math.ceil(math.sqrt(len(byte_array))))\n    pad_size = (width * width) - len(byte_array)\n    \n    R_pad = np.pad(R, (0, pad_size), 'constant').reshape(width, width)\n    G_pad = np.pad(G, (0, pad_size), 'constant').reshape(width, width)\n    B_pad = np.pad(B, (0, pad_size), 'constant').reshape(width, width)\n    \n    img_bgr = np.dstack((B_pad, G_pad, R_pad))\n    final_img = cv2.resize(img_bgr, IMAGE_SIZE, interpolation=cv2.INTER_AREA)\n    cv2.imwrite(out_path, final_img)\n\n","metadata":{"execution":{"iopub.execute_input":"2026-04-08T10:49:24.175528Z","iopub.status.busy":"2026-04-08T10:49:24.175062Z","iopub.status.idle":"2026-04-08T10:49:24.189088Z","shell.execute_reply":"2026-04-08T10:49:24.187928Z"},"papermill":{"duration":0.019781,"end_time":"2026-04-08T10:49:24.191306+00:00","exception":false,"start_time":"2026-04-08T10:49:24.171525+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1fc1ee3d","cell_type":"code","source":"# .asm\ndef process_asm_markov(asm_path, out_path):\n    if os.path.exists(out_path): return\n    transition_matrix = np.zeros((256, 256), dtype=np.float32)\n    prev_idx = None\n    \n    try:\n        with open(asm_path, 'r', encoding='latin1', errors='ignore') as f:\n            for line in f:\n                parts = line.split()\n                if len(parts) > 1:\n                    opcode = parts[1].lower()\n                    if not opcode.isalnum(): continue\n                    \n                    # Băm Opcode thành tọa độ pixel\n                    curr_idx = zlib.crc32(opcode.encode('utf-8')) % 256\n                    \n                    if prev_idx is not None:\n                        transition_matrix[prev_idx, curr_idx] += 1\n                    prev_idx = curr_idx\n    except Exception: return\n\n    # Chuẩn hóa theo thang Logarit\n    matrix_log = np.log1p(transition_matrix)\n    max_val = np.max(matrix_log)\n    \n    if max_val > 0:\n        img_markov = (matrix_log / max_val) * 255.0\n    else:\n        img_markov = matrix_log\n        \n    cv2.imwrite(out_path, img_markov.astype(np.uint8))","metadata":{"execution":{"iopub.execute_input":"2026-04-08T10:49:24.197651Z","iopub.status.busy":"2026-04-08T10:49:24.196534Z","iopub.status.idle":"2026-04-08T10:49:24.205856Z","shell.execute_reply":"2026-04-08T10:49:24.204332Z"},"papermill":{"duration":0.014923,"end_time":"2026-04-08T10:49:24.208243+00:00","exception":false,"start_time":"2026-04-08T10:49:24.19332+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"950753a8","cell_type":"code","source":"def save_checkpoint_and_clear():\n    \"\"\"Ghi nối (append) các ảnh PNG mới vào 2 file ZIP tổng và xóa ảnh thô\"\"\"\n    \n    # 1. Xử lý thư mục Bytes\n    if len(os.listdir(OUT_BYTES_DIR)) > 0:\n        with zipfile.ZipFile(FINAL_BYTES_ZIP, 'a', zipfile.ZIP_DEFLATED) as zipf:\n            for filename in os.listdir(OUT_BYTES_DIR):\n                filepath = os.path.join(OUT_BYTES_DIR, filename)\n                zipf.write(filepath, arcname=filename) \n                os.remove(filepath)\n                \n    # 2. Xử lý thư mục ASM\n    if len(os.listdir(OUT_ASM_DIR)) > 0:\n        with zipfile.ZipFile(FINAL_ASM_ZIP, 'a', zipfile.ZIP_DEFLATED) as zipf:\n            for filename in os.listdir(OUT_ASM_DIR):\n                filepath = os.path.join(OUT_ASM_DIR, filename)\n                zipf.write(filepath, arcname=filename)\n                os.remove(filepath)\n\ndef run_pipeline():\n    df_labels = pd.read_csv(LABELS_CSV)\n    all_ids = df_labels['Id'].tolist()[START_INDEX:END_INDEX]\n    total_files = len(all_ids)\n    \n    for i in tqdm(range(0, total_files, BATCH_SIZE), desc=\"Processing Batches\"):\n        batch_ids = all_ids[i:i+BATCH_SIZE]\n        \n        current_batch_start = START_INDEX + i\n        current_batch_end = START_INDEX + i + len(batch_ids)\n        \n        list_file_path = os.path.join(WORKING_DIR, 'batch_list.txt')\n        with open(list_file_path, 'w') as f:\n            for m_id in batch_ids:\n                f.write(f\"{m_id}.bytes\\n\")\n                f.write(f\"{m_id}.asm\\n\")\n                \n        cmd = f\"7z e {TRAIN_ZIP} -o{TEMP_DIR} -ir@{list_file_path} -y\"\n        subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)\n        \n        for m_id in batch_ids:\n            bytes_in = os.path.join(TEMP_DIR, f\"{m_id}.bytes\")\n            asm_in = os.path.join(TEMP_DIR, f\"{m_id}.asm\")\n            \n            bytes_out = os.path.join(OUT_BYTES_DIR, f\"{m_id}.png\")\n            asm_out = os.path.join(OUT_ASM_DIR, f\"{m_id}.png\")\n            \n            if os.path.exists(bytes_in):\n                process_bytes_rgb(bytes_in, bytes_out)\n                os.remove(bytes_in) \n                \n            if os.path.exists(asm_in):\n                process_asm_markov(asm_in, asm_out)\n                os.remove(asm_in) \n        \n        save_checkpoint_and_clear()\n        gc.collect()\n        \n        # Dùng tqdm.write để in tiến độ mà không làm vỡ giao diện thanh chạy\n        tqdm.write(f\"✓ Đã xử lý và nén xong: {current_batch_end}/{total_files} ảnh\")","metadata":{"execution":{"iopub.execute_input":"2026-04-08T10:49:24.214319Z","iopub.status.busy":"2026-04-08T10:49:24.213961Z","iopub.status.idle":"2026-04-08T10:49:24.22634Z","shell.execute_reply":"2026-04-08T10:49:24.225183Z"},"papermill":{"duration":0.018378,"end_time":"2026-04-08T10:49:24.228623+00:00","exception":false,"start_time":"2026-04-08T10:49:24.210245+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"ed77dca1","cell_type":"code","source":"if __name__ == '__main__':\n    run_pipeline()\n    print(f\"\\nHOÀN TẤT: Toàn bộ dữ liệu cụm [{START_INDEX} - {END_INDEX}] đã được gộp gọn vào 2 file ZIP!\")\n    \n    # Dọn sạch rác\n    !rm -rf {TEMP_DIR} {OUT_BYTES_DIR} {OUT_ASM_DIR} batch_list.txt","metadata":{"execution":{"iopub.execute_input":"2026-04-08T10:49:24.23433Z","iopub.status.busy":"2026-04-08T10:49:24.233737Z","iopub.status.idle":"2026-04-08T17:20:45.453642Z","shell.execute_reply":"2026-04-08T17:20:45.452126Z"},"papermill":{"duration":23481.227471,"end_time":"2026-04-08T17:20:45.458053+00:00","exception":false,"start_time":"2026-04-08T10:49:24.230582+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}