{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, pandas as pd, numpy as np, subprocess, re, time, torch, gc\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom PIL import Image\nfrom collections import Counter\nfrom torchvision import transforms, models\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom sklearn.model_selection import train_test_split\n\n# --- 1. CẤU HÌNH HỆ THỐNG ---\nTRAIN_7Z = '/kaggle/input/competitions/malware-classification/train.7z'\nTEST_7Z = '/kaggle/input/competitions/malware-classification/test.7z'\nTRAIN_LABELS = '/kaggle/input/competitions/malware-classification/trainLabels.csv'\nTEMP_DIR = '/kaggle/working/temp_data'\nSUBMISSION_FILE = 'submission_final.csv'\nos.makedirs(TEMP_DIR, exist_ok=True)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nopcodes = ['jmp', 'mov', 'retf', 'push', 'pop', 'add', 'sub', 'mul', 'div', 'and', 'or', 'xor', 'cmp', 'test', 'call']\nkeywords = ['db', 'dw', 'dd', 'offset', 'api', '.dll', 'std::']\nn_asm_features = len(opcodes) + len(keywords)\n\n# --- 2. HÀM TRÍCH XUẤT ĐẶC TRƯNG ---\ndef extract_features_safe(fid):\n    b_path = os.path.join(TEMP_DIR, f\"{fid}.bytes\")\n    a_path = os.path.join(TEMP_DIR, f\"{fid}.asm\")\n    img_t = torch.zeros((1, 1, 128, 128))\n    asm_v = [0 for _ in range(n_asm_features)] \n    \n    if os.path.exists(b_path):\n        try:\n            with open(b_path, 'r') as f:\n                content = f.read(1000000)\n                hex_list = [int(x, 16) for x in content.split() if len(x)==2 and x!='??']\n            if hex_list:\n                d = np.array(hex_list, dtype=np.uint8)\n                side = int(len(d)**0.5)\n                if side > 10:\n                    img = Image.fromarray(d[:side*side].reshape((side, side))).convert('L').resize((128, 128))\n                    img_t = transforms.ToTensor()(img).unsqueeze(0)\n        except: pass\n\n    if os.path.exists(a_path):\n        try:\n            if os.path.getsize(a_path) < 150 * 1024 * 1024:\n                with open(a_path, 'r', encoding='latin-1') as f:\n                    content = f.read().lower()\n                    tokens = Counter(re.findall(r'\\b[a-z]+(?:::)?\\b', content))\n                    asm_v = [tokens.get(o, 0) for o in opcodes] + [tokens.get(k, 0) for k in keywords]\n        except: pass\n    return img_t, asm_v\n\n# --- 3. TRÍCH XUẤT DỮ LIỆU ---\nprint(\">>> BẮT ĐẦU TRÍCH XUẤT DỮ LIỆU...\")\nlabels_df = pd.read_csv(TRAIN_LABELS)\ntrain_ids = labels_df['Id'].values\nX_asm_all, X_img_all, Y_all = [], [], []\n\nT_BATCH = 100 \nfor i in range(0, len(train_ids), T_BATCH):\n    batch = train_ids[i : i+T_BATCH]\n    \n    with open('train_list.txt', 'w') as f:\n        for fid in batch:\n            f.write(f\"train/{fid}.bytes\\ntrain/{fid}.asm\\n\")\n            \n    subprocess.run(f\"7z e {TRAIN_7Z} -o{TEMP_DIR} -i@train_list.txt -y > /dev/null\", shell=True)\n    \n    for fid in batch:\n        img, asm = extract_features_safe(fid)\n        \n        try:\n            label_row = labels_df[labels_df['Id'] == fid]\n            if not label_row.empty:\n                # ĐÃ FIX LỖI .iloc[0] Ở ĐÂY\n                label_val = int(label_row['Class'].iloc[0]) - 1\n                X_img_all.append(img) \n                X_asm_all.append(asm)\n                Y_all.append(label_val)\n        except:\n            continue\n    \n    subprocess.run(f\"rm -rf {TEMP_DIR}/*\", shell=True)\n    if (i+T_BATCH) % 1000 == 0:\n        print(f\"Tiến độ: {min(i+T_BATCH, len(train_ids))} file\")\n\nif len(X_img_all) == 0:\n    raise ValueError(\"❌ Không có dữ liệu nào được trích xuất. Hãy kiểm tra đường dẫn TRAIN_7Z!\")\n\nX_asm_all = np.array(X_asm_all, dtype=np.float32)\nX_img_all = torch.cat(X_img_all, dim=0)\nY_all = np.array(Y_all).astype(np.int64)\n\n# --- 4. CHIA DỮ LIỆU 80% TRAIN - 20% VALIDATION ---\nprint(\">>> ĐANG CHIA DỮ LIỆU 80/20...\")\nidx_train, idx_val = train_test_split(\n    np.arange(len(Y_all)), \n    test_size=0.2, \n    random_state=42, \n    stratify=Y_all\n)\n\nX_asm_train, X_asm_val = X_asm_all[idx_train], X_asm_all[idx_val]\nX_img_train, X_img_val = X_img_all[idx_train], X_img_all[idx_val]\nY_train, Y_val = Y_all[idx_train], Y_all[idx_val]\n\ndel X_asm_all, X_img_all, Y_all\ngc.collect()\nprint(f\"✅ Train set: {len(Y_train)} mẫu | Val set: {len(Y_val)} mẫu\")\n\n# --- 5. HUẤN LUYỆN MODEL ---\nprint(\">>> ĐANG LUYỆN XGBOOST...\")\nimport xgboost as xgb\nxgb_model = xgb.XGBClassifier(\n    tree_method='hist', \n    device='cuda' if torch.cuda.is_available() else 'cpu', \n    n_estimators=300\n)\nxgb_model.fit(X_asm_train, Y_train)\n\nprint(\">>> ĐANG LUYỆN RESNET (10 EPOCHS)...\")\nmodel_resnet = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)\nmodel_resnet.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\nmodel_resnet.fc = nn.Linear(512, 9)\nmodel_resnet.to(device)\n\ntrain_loader = DataLoader(TensorDataset(X_img_train, torch.tensor(Y_train)), batch_size=64, shuffle=True)\noptimizer = optim.Adam(model_resnet.parameters(), lr=0.001)\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(10):\n    model_resnet.train()\n    for inputs, labels in train_loader:\n        inputs, labels = inputs.to(device), labels.to(device)\n        optimizer.zero_grad()\n        loss = criterion(model_resnet(inputs), labels)\n        loss.backward()\n        optimizer.step()\n    print(f\"Epoch {epoch+1} hoàn tất.\")\n\n# Đánh giá trên 20% validation\nmodel_resnet.eval()\nwith torch.no_grad():\n    res_val = F.softmax(model_resnet(X_img_val.to(device)), dim=1).cpu().numpy()\n    xgb_val = xgb_model.predict_proba(X_asm_val)\n    val_acc = np.mean(np.argmax(0.4 * res_val + 0.6 * xgb_val, axis=1) == Y_val)\n    print(f\"✅ Accuracy trên tập Validation: {val_acc:.4f}\")\n\n# --- 6. DỰ ĐOÁN TẬP TEST ---\nprint(\">>> ĐANG DỰ ĐOÁN TẬP TEST...\")\nraw_test = subprocess.check_output(f\"7z l {TEST_7Z} | grep .bytes | awk '{{print $NF}}'\", shell=True).decode().splitlines()\ntest_ids = [os.path.basename(f).replace('.bytes', '') for f in raw_test if len(os.path.basename(f)) > 15]\n\nif os.path.exists(SUBMISSION_FILE): os.remove(SUBMISSION_FILE)\n\nmodel_resnet.eval()\nfor i in range(0, len(test_ids), 100):\n    batch = test_ids[i : i+100]\n    with open('test_batch.txt', 'w') as f:\n        for fid in batch: f.write(f\"test/{fid}.bytes\\ntest/{fid}.asm\\n\")\n    \n    subprocess.run(f\"7z e {TEST_7Z} -o{TEMP_DIR} -i@test_batch.txt -y > /dev/null\", shell=True)\n    \n    # ĐÃ FIX THỤT LỀ CHO ĐOẠN XỬ LÝ BÊN TRONG VÒNG LẶP TEST\n    batch_probs = []\n    for fid in batch:\n        img_t, asm_f = extract_features_safe(fid)\n        with torch.no_grad():\n            p_resnet = F.softmax(model_resnet(img_t.to(device)), dim=1).cpu().numpy().flatten()\n        p_xgb = xgb_model.predict_proba(np.array([asm_f])).flatten()\n        \n        final_xgb = [0.0] * 9\n        for idx in range(min(len(p_xgb), 9)): \n            final_xgb[idx] = p_xgb[idx]\n            \n        combined = (0.4 * p_resnet + 0.6 * np.array(final_xgb)).tolist()\n        batch_probs.append([fid] + combined)\n    \n    cols = ['Id'] + [f'Prediction{j}' for j in range(1, 10)]\n    pd.DataFrame(batch_probs, columns=cols).to_csv(SUBMISSION_FILE, mode='a', header=not os.path.exists(SUBMISSION_FILE), index=False)\n    \n    subprocess.run(f\"rm -rf {TEMP_DIR}/*\", shell=True)\n    if (i+100) % 500 == 0: \n        print(f\"Tiến độ Test: {min(i+100, len(test_ids))}/{len(test_ids)}\")\n        gc.collect()\n        torch.cuda.empty_cache()\n\nprint(f\"🏁 Xong! File nộp bài: {SUBMISSION_FILE}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-04-26T03:24:49.236897Z","iopub.execute_input":"2026-04-26T03:24:49.237182Z"}},"outputs":[],"execution_count":null}]}