{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665},{"sourceType":"kernelVersion","sourceId":316446747}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# PHASE 2 — Hybrid Architecture: Adaptive CNN + XGBoost\n# Input : train_tabular_features_batch_*.csv (280 features)\n#         train_malware_images/*.png (ảnh gốc aspect ratio)\n# Target: OOF accuracy >= 99%, log-loss <= 0.025\n#\n# Pipeline:\n#   1. Load 280 tabular features\n#   2. Train Adaptive CNN (grayscale, batch_size=1) → 256-dim embed\n#   3. Optuna (15 trials) → best XGBoost params\n#   4. Stratified 5-Fold CV (Zero Data Leakage)\n#   5. Production model trên 100% data\n# ============================================================\n\n# !pip install optuna xgboost imbalanced-learn -q\n\nimport os, gc, glob, warnings\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport optuna\nimport xgboost as xgb\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import accuracy_score, log_loss, classification_report\nfrom imblearn.over_sampling import SMOTE\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nwarnings.filterwarnings('ignore')\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\n# ── Config ────────────────────────────────────────────────\nTABULAR_DIR = \"/kaggle/input/notebooks/khanq1504/malware-phase1\"   # ← sửa thành output Phase 1 của bạn\nIMAGE_DIR   = os.path.join(TABULAR_DIR, \"train_malware_images\")\nLABEL_PATH  = \"/kaggle/input/competitions/malware-classification/trainLabels.csv\"\nDEVICE      = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nSEED        = 42\nprint(f\"🖥️  Device: {DEVICE}\")\n\n\n# ════════════════════════════════════════════════════════════\n# 1. LOAD TABULAR FEATURES (280 features từ Phase 1)\n# ════════════════════════════════════════════════════════════\nprint(\"\\n📂 Load tabular features...\")\nall_csvs    = glob.glob(os.path.join(TABULAR_DIR, \"train_tabular_features_batch_*.csv\"))\nif not all_csvs:\n    # Fallback: thử tìm theo pattern khác\n    all_csvs = glob.glob(os.path.join(\"/kaggle/working\", \"train_tabular_features_batch_*.csv\"))\n\nprint(f\"   Tìm thấy {len(all_csvs)} batch CSV files\")\ndf_features = pd.concat([pd.read_csv(f) for f in sorted(all_csvs)], ignore_index=True)\ndf_labels   = pd.read_csv(LABEL_PATH)\n\ndf_full = (pd.merge(df_features, df_labels, on='Id', how='inner')\n             .sort_values('Id')\n             .reset_index(drop=True))\n\n# Drop rows có NaN trong features\nfeat_cols = [c for c in df_full.columns if c not in ['Id', 'Class']]\ndf_full   = df_full.dropna(subset=feat_cols)\n\n# Encode labels\nencoder    = LabelEncoder()\ny          = encoder.fit_transform(df_full['Class'])\nnum_classes = len(np.unique(y))\njoblib.dump(encoder, '/kaggle/working/label_encoder.pkl')\n\nX_tabular    = df_full[feat_cols].values.astype(np.float32)\nimage_ids    = df_full['Id'].tolist()\nfeature_names = feat_cols\n\nprint(f\"   Samples : {len(y)}\")\nprint(f\"   Features: {X_tabular.shape[1]} tabular features\")\nprint(f\"   Classes : {num_classes} — {list(encoder.classes_)}\")\n\ndel df_full, df_features\ngc.collect()\n\n\n# ════════════════════════════════════════════════════════════\n# 2. ADAPTIVE CNN — Grayscale, NO resize, batch_size=1\n# ════════════════════════════════════════════════════════════\n\nclass MalwareImageDataset(Dataset):\n    \"\"\"Dataset đọc ảnh grayscale gốc (không resize).\"\"\"\n    def __init__(self, image_dir, image_ids, labels=None):\n        self.image_dir = image_dir\n        self.image_ids = image_ids\n        self.labels    = labels\n        self.transform = transforms.Compose([transforms.ToTensor()])\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        import cv2\n        img_path = os.path.join(self.image_dir, f\"{self.image_ids[idx]}.png\")\n        image    = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n        if image is None:\n            image = np.zeros((224, 224), dtype=np.uint8)\n        image = self.transform(image)   # shape: [1, H, W]\n        if self.labels is not None:\n            return image, torch.tensor(self.labels[idx], dtype=torch.long)\n        return image\n\n\nclass AdaptiveMalwareCNN(nn.Module):\n    \"\"\"\n    CNN với AdaptiveAvgPool2d — nhận ảnh bất kỳ kích thước.\n    Input : [1, H, W] grayscale\n    Output: 256-dim embedding (dùng cho XGBoost)\n    \"\"\"\n    def __init__(self, num_classes=9):\n        super().__init__()\n        self.features = nn.Sequential(\n            nn.Conv2d(1, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2, 2),\n            nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2, 2),\n            nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2, 2),\n        )\n        self.adaptive_pool = nn.AdaptiveAvgPool2d((4, 4))\n        self.flatten_size  = 128 * 4 * 4  # = 2048\n\n        self.classifier = nn.Sequential(\n            nn.Linear(self.flatten_size, 256), nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(256, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.adaptive_pool(x)\n        x = x.view(x.size(0), -1)\n        return self.classifier(x)\n\n    def extract_features(self, x):\n        \"\"\"Trả về 256-dim embedding (trước lớp phân loại cuối).\"\"\"\n        x = self.features(x)\n        x = self.adaptive_pool(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier[0](x)   # Linear 2048→256\n        x = self.classifier[1](x)   # ReLU\n        return x\n\n\n# Damped Inverse Frequency Weighting\nclass_counts   = np.bincount(y)\ndamped_weights = np.log(len(y) / class_counts + 1.2)\ntensor_weights = torch.tensor(damped_weights, dtype=torch.float32).to(DEVICE)\nprint(f\"\\n⚖️  Damped weights: {[f'{w:.3f}' for w in damped_weights]}\")\n\n# Train trên 80% data (đủ để học visual pattern, tiết kiệm thời gian)\nprint(\"\\n🚀 Train Adaptive CNN (2 epochs, batch_size=1)...\")\ntrain_idx, _, y_tr, _ = train_test_split(\n    range(len(y)), y, test_size=0.2, random_state=SEED, stratify=y\n)\ntrain_ids     = [image_ids[i] for i in train_idx]\ntrain_dataset = MalwareImageDataset(IMAGE_DIR, train_ids, y_tr)\ntrain_loader  = DataLoader(train_dataset, batch_size=1, shuffle=True,\n                           num_workers=2, pin_memory=True)\n\ncnn_model = AdaptiveMalwareCNN(num_classes).to(DEVICE)\ncriterion = nn.CrossEntropyLoss(weight=tensor_weights)\noptimizer = optim.Adam(cnn_model.parameters(), lr=1e-3)\n\ncnn_model.train()\nfor epoch in range(2):\n    running_loss = 0.0\n    for inputs, labels in train_loader:\n        inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)\n        optimizer.zero_grad()\n        outputs = cnn_model(inputs)\n        loss    = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n    print(f\"   Epoch {epoch+1}/2 — Loss: {running_loss/len(train_loader):.4f}\")\n\ntorch.save(cnn_model.state_dict(), '/kaggle/working/adaptive_cnn_weights.pth')\nprint(\"   ✅ CNN weights saved\")\n\n\n# ── Extract embeddings toàn bộ dataset ────────────────────\nprint(\"\\n🔍 Extract 256-dim embeddings (toàn bộ dataset)...\")\ncnn_model.eval()\nfull_dataset  = MalwareImageDataset(IMAGE_DIR, image_ids)\nfull_loader   = DataLoader(full_dataset, batch_size=1, shuffle=False,\n                           num_workers=2, pin_memory=True)\n\nX_cnn_features = []\nwith torch.no_grad():\n    for inputs in full_loader:\n        inputs = inputs.to(DEVICE)\n        feat   = cnn_model.extract_features(inputs).cpu().numpy()\n        X_cnn_features.append(feat[0])\n\nX_cnn_features = np.array(X_cnn_features)\nprint(f\"   CNN Embeddings: {X_cnn_features.shape}\")\n\n# Hybrid features: CNN (256) + Tabular (280) = 536 features\nX_hybrid_raw = np.hstack((X_cnn_features, X_tabular))\nprint(f\"   Hybrid features: {X_hybrid_raw.shape}  (256 CNN + {X_tabular.shape[1]} tabular)\")\n\ndel cnn_model, train_dataset, train_loader, full_dataset, full_loader\ntorch.cuda.empty_cache()\ngc.collect()\n\n\n# ════════════════════════════════════════════════════════════\n# 3. OPTUNA — Tìm hyperparams XGBoost (15 trials)\n# ════════════════════════════════════════════════════════════\nprint(\"\\n🔎 Optuna Bayesian Search (15 trials)...\")\n\ndef objective(trial):\n    X_tr_r, X_va_r, y_tr, y_va = train_test_split(\n        X_hybrid_raw, y, test_size=0.2, random_state=SEED, stratify=y\n    )\n    scaler    = StandardScaler()\n    X_tr_s    = scaler.fit_transform(X_tr_r)\n    X_va_s    = scaler.transform(X_va_r)\n\n    smote     = SMOTE(k_neighbors=3, random_state=SEED)\n    X_tr_sm, y_tr_sm = smote.fit_resample(X_tr_s, y_tr)\n\n    params = {\n        'objective'       : 'multi:softprob',\n        'num_class'       : num_classes,\n        'eval_metric'     : 'mlogloss',\n        'tree_method'     : 'hist',\n        'device'          : 'cuda' if torch.cuda.is_available() else 'cpu',\n        'max_depth'       : trial.suggest_int('max_depth', 3, 8),\n        'learning_rate'   : trial.suggest_float('learning_rate', 0.01, 0.1),\n        'subsample'       : trial.suggest_float('subsample', 0.6, 0.9),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 0.9),\n        'reg_alpha'       : trial.suggest_float('reg_alpha', 1e-3, 10.0, log=True),\n        'reg_lambda'      : trial.suggest_float('reg_lambda', 1e-3, 10.0, log=True),\n    }\n    bst   = xgb.train(params, xgb.DMatrix(X_tr_sm, label=y_tr_sm), num_boost_round=150)\n    preds = bst.predict(xgb.DMatrix(X_va_s))\n    return log_loss(y_va, preds)\n\nstudy = optuna.create_study(\n    direction='minimize',\n    sampler=optuna.samplers.TPESampler(seed=SEED)\n)\nstudy.optimize(objective, n_trials=15, show_progress_bar=True)\n\nbest_params = study.best_params\nbest_params.update({\n    'objective'  : 'multi:softprob',\n    'num_class'  : num_classes,\n    'tree_method': 'hist',\n    'eval_metric': 'mlogloss',\n})\nprint(f\"\\n✅ Best Log-Loss (Optuna): {study.best_value:.4f}\")\nprint(f\"   Best Params: {study.best_params}\")\n\n\n# ════════════════════════════════════════════════════════════\n# 4. STRATIFIED 5-FOLD CV — Zero Data Leakage\n# ════════════════════════════════════════════════════════════\nprint(\"\\n\" + \"=\"*60)\nprint(\"📊 Stratified 5-Fold CV (Zero Data Leakage)\")\nprint(\"=\"*60)\n\nskf       = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)\noof_preds = np.zeros((len(X_hybrid_raw), num_classes))\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X_hybrid_raw, y), 1):\n    X_tr_r, y_tr = X_hybrid_raw[train_idx], y[train_idx]\n    X_va_r, y_va = X_hybrid_raw[val_idx],   y[val_idx]\n\n    # ⚠️ ZERO DATA LEAKAGE: fit chỉ trên train fold\n    scaler = StandardScaler()\n    X_tr_s = scaler.fit_transform(X_tr_r)\n    X_va_s = scaler.transform(X_va_r)\n\n    smote  = SMOTE(k_neighbors=3, random_state=SEED)\n    X_tr_sm, y_tr_sm = smote.fit_resample(X_tr_s, y_tr)\n\n    xgb_model = xgb.XGBClassifier(\n        n_estimators=400,\n        early_stopping_rounds=30,\n        **best_params\n    )\n    xgb_model.fit(\n        X_tr_sm, y_tr_sm,\n        eval_set=[(X_va_s, y_va)],\n        verbose=False\n    )\n    oof_preds[val_idx] = xgb_model.predict_proba(X_va_s)\n    fold_ll = log_loss(y_va, oof_preds[val_idx])\n    fold_acc = accuracy_score(y_va, np.argmax(oof_preds[val_idx], axis=1))\n    print(f\"   Fold {fold}/5 — Accuracy: {fold_acc:.4f}  Log-Loss: {fold_ll:.4f}\")\n\noof_labels = np.argmax(oof_preds, axis=1)\noof_acc    = accuracy_score(y, oof_labels)\noof_ll     = log_loss(y, oof_preds)\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"📊 OOF RESULTS:\")\nprint(f\"   True Generalized Accuracy : {oof_acc*100:.2f}%\")\nprint(f\"   True Generalized Log-Loss : {oof_ll:.4f}\")\nprint(\"=\"*60)\n\n\n# ════════════════════════════════════════════════════════════\n# 5. PRODUCTION MODEL — Train trên 100% data\n# ════════════════════════════════════════════════════════════\nprint(\"\\n🏭 Train Production Model (100% data)...\")\n\nfinal_scaler          = StandardScaler()\nX_hybrid_scaled_final = final_scaler.fit_transform(X_hybrid_raw)\njoblib.dump(final_scaler, '/kaggle/working/tabular_scaler_final.pkl')\n\nsmote_final   = SMOTE(k_neighbors=3, random_state=SEED)\nX_sm, y_sm    = smote_final.fit_resample(X_hybrid_scaled_final, y)\nprint(f\"   Sau SMOTE: {len(y)} → {len(y_sm)} samples\")\n\nfinal_model = xgb.XGBClassifier(n_estimators=400, **best_params)\nfinal_model.fit(X_sm, y_sm, verbose=False)\nfinal_model.save_model('/kaggle/working/xgboost_model_final.json')\nprint(\"   ✅ Saved: xgboost_model_final.json, tabular_scaler_final.pkl\")\n\n\n# ── Classification Report ─────────────────────────────────\nprint(\"\\n📋 Classification Report (OOF):\")\nclass_display = [f\"Class_{c}\" for c in encoder.classes_]\nprint(classification_report(y, oof_labels, target_names=class_display))\n\n# Confusion Matrix\ncm = confusion_matrix(y, oof_labels)\nplt.figure(figsize=(11, 9))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\n            xticklabels=encoder.classes_, yticklabels=encoder.classes_,\n            linewidths=0.5)\nplt.title(f\"Confusion Matrix — OOF 5-Fold (Acc={oof_acc*100:.2f}%)\",\n          fontsize=13, fontweight='bold')\nplt.ylabel(\"True\"); plt.xlabel(\"Predicted\")\nplt.tight_layout()\nplt.savefig('/kaggle/working/confusion_matrix.png', dpi=130)\nplt.show()\n\n# Feature Importance (top 15)\nprint(\"\\n🔍 Top 15 Feature Importances (Gain):\")\nimportance  = final_model.get_booster().get_score(importance_type='gain')\nsorted_imp  = sorted(importance.items(), key=lambda x: x[1], reverse=True)\nn_cnn_feats = X_cnn_features.shape[1]  # 256\n\nfor rank, (feat, gain) in enumerate(sorted_imp[:15], 1):\n    idx = int(feat[1:])\n    if idx < n_cnn_feats:\n        name = f\"CNN_Embed_{idx}\"\n    else:\n        tab_idx = idx - n_cnn_feats\n        name = feature_names[tab_idx] if tab_idx < len(feature_names) else f\"tab_{tab_idx}\"\n    print(f\"   {rank:2}. {name:<25} Gain: {gain:.2f}\")\n\n# Summary\nprint(\"\\n\" + \"=\"*60)\nprint(\"🏆 PHASE 2 HOÀN TẤT:\")\nprint(f\"   OOF Accuracy  : {oof_acc*100:.2f}%\")\nprint(f\"   OOF Log-Loss  : {oof_ll:.4f}\")\nprint(f\"   Target        : >= 99.00%,  <= 0.025\")\nif oof_acc >= 0.99:\n    print(\"   🎉 ĐẠT MỤC TIÊU!\")\nelse:\n    print(f\"   📈 Còn thiếu  : {(0.99 - oof_acc)*100:.2f}%\")\nprint(\"=\"*60)\nprint(\"\\n📁 Artifacts:\")\nprint(\"   /kaggle/working/adaptive_cnn_weights.pth\")\nprint(\"   /kaggle/working/xgboost_model_final.json\")\nprint(\"   /kaggle/working/tabular_scaler_final.pkl\")\nprint(\"   /kaggle/working/label_encoder.pkl\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-05-07T01:29:03.282145Z","iopub.execute_input":"2026-05-07T01:29:03.282445Z","execution_failed":"2026-05-07T14:15:55.077Z"}},"outputs":[],"execution_count":null}]}