{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":4117,"databundleVersionId":46665},{"sourceType":"datasetVersion","sourceId":15601117,"datasetId":9966584,"databundleVersionId":16534249},{"sourceType":"datasetVersion","sourceId":15813534,"datasetId":10092136,"databundleVersionId":16761687}],"dockerImageVersionId":31329,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"title":"Exp 05: Optimized Byte and ASM Features with 5-Fold XGBoost Cross-Validation"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Exp 05: Optimized Byte and ASM Features with 5-Fold XGBoost Cross-Validation\n\nThis experiment evaluates a non-CNN malware-classification pipeline based\nentirely on optimized handcrafted numeric features derived from the malware\nbyte and assembly representations.\n\nThe model uses XGBoost with stratified five-fold cross-validation. Each fold\nfits its own `MinMaxScaler`, trains an independent multiclass XGBoost model,\nand produces out-of-fold validation probabilities. Test predictions are\ngenerated by averaging the probability outputs from all five fold models.\n\n| Component | Configuration |\n|---|---|\n| Labeled samples | 10,868 |\n| Number of classes | 9 |\n| Input representation | Optimized byte and ASM numeric features |\n| Image/CNN branch | Not used |\n| Validation strategy | Stratified 5-fold cross-validation |\n| Fold preprocessing | Independent min–max scaler |\n| Model | XGBoost multiclass classifier |\n| Maximum estimators | 1,500 |\n| Learning rate | 0.03 |\n| Maximum depth | 8 |\n| Row subsampling | 0.8 |\n| Column subsampling | 0.8 |\n| Early stopping | 100 rounds |\n| Training hardware | Two T4 GPUs with fold-level parallelism |\n| Prediction aggregation | Mean probability across five folds |\n| Overall OOF log loss | 0.007701568294 |\n| Reported OOF accuracy | 1.00 after rounding |","metadata":{}},{"cell_type":"markdown","source":"## 1. Dependencies\n","metadata":{}},{"cell_type":"code","source":"import os\n\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.metrics import classification_report, log_loss\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import MinMaxScaler\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:46:55.741064Z","iopub.execute_input":"2026-04-21T16:46:55.741713Z","iopub.status.idle":"2026-04-21T16:46:57.023912Z","shell.execute_reply.started":"2026-04-21T16:46:55.741674Z","shell.execute_reply":"2026-04-21T16:46:57.023085Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Optimized Feature Dataset Assembly\n","metadata":{}},{"cell_type":"code","source":"TRAIN_LABELS_PATH = (\n    '/kaggle/input/competitions/'\n    'malware-classification/trainLabels.csv'\n)\nTRAIN_FEATURES_CSV = (\n    '/kaggle/input/datasets/swannessie/'\n    'optimized-feature/'\n    'input_ultimate_malware_features_FULL.csv'\n)\nTEST_FEATURES_CSV = (\n    '/kaggle/input/datasets/swannessie/'\n    'optimized-feature/'\n    'ultimate_malware_features_TEST.csv'\n)\n\nlabels_df = pd.read_csv(TRAIN_LABELS_PATH)\nfeatures_df = pd.read_csv(\n    TRAIN_FEATURES_CSV\n).fillna(0)\n\ndf_train = pd.merge(\n    labels_df,\n    features_df,\n    left_on='Id',\n    right_on='ID',\n    how='inner'\n)\n\nnumeric_cols = features_df.select_dtypes(\n    include=[np.number]\n).columns.tolist()\nfeature_columns = [\n    column\n    for column in numeric_cols\n    if column not in ['ID', 'Id', 'Class']\n]\n\nX = df_train[feature_columns].values\ny = df_train['Class'].values - 1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:46:58.958954Z","iopub.execute_input":"2026-04-21T16:46:58.959402Z","iopub.status.idle":"2026-04-21T16:47:04.980309Z","shell.execute_reply.started":"2026-04-21T16:46:58.959373Z","shell.execute_reply":"2026-04-21T16:47:04.979533Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Exploratory Feature Analysis\n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set_theme(style='whitegrid')\n\nprint(\"Generating exploratory feature plots...\")\n\nplt.figure(figsize=(10, 5))\nax = sns.countplot(\n    data=df_train,\n    x='Class',\n    hue='Class',\n    palette='viridis',\n    legend=False\n)\nplt.title(\n    'Malware Class Distribution',\n    fontsize=14,\n    fontweight='bold'\n)\nplt.xlabel('Malware Class', fontsize=12)\nplt.ylabel('Sample Count', fontsize=12)\n\nfor patch in ax.patches:\n    ax.annotate(\n        f'{int(patch.get_height())}',\n        (\n            patch.get_x() + patch.get_width() / 2,\n            patch.get_height()\n        ),\n        ha='center',\n        va='bottom',\n        fontsize=10,\n        xytext=(0, 5),\n        textcoords='offset points'\n    )\n\nplt.show()\n\ntop_features = feature_columns[:4]\nfig, axes = plt.subplots(\n    2,\n    2,\n    figsize=(14, 10)\n)\naxes = axes.flatten()\n\nfor index, column in enumerate(top_features):\n    sns.histplot(\n        df_train[column],\n        bins=50,\n        kde=True,\n        ax=axes[index],\n        color='teal'\n    )\n    axes[index].set_title(\n        f'Distribution: {column}',\n        fontsize=12\n    )\n    axes[index].set_xlabel('Value')\n    axes[index].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n\nplt.figure(figsize=(10, 8))\nsubset_features = feature_columns[:10]\ncorr_matrix = df_train[\n    subset_features\n].corr()\n\nsns.heatmap(\n    corr_matrix,\n    annot=True,\n    fmt='.2f',\n    cmap='coolwarm',\n    cbar=True,\n    square=True,\n    linewidths=0.5\n)\nplt.title(\n    'Correlation Matrix of the First 10 Features',\n    fontsize=14,\n    fontweight='bold'\n)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:47:07.299499Z","iopub.execute_input":"2026-04-21T16:47:07.299783Z","iopub.status.idle":"2026-04-21T16:47:09.185772Z","shell.execute_reply.started":"2026-04-21T16:47:07.299753Z","shell.execute_reply":"2026-04-21T16:47:09.185037Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Parallel Five-Fold XGBoost Training\n","metadata":{}},{"cell_type":"code","source":"from joblib import Parallel, delayed\n\nn_splits = 5\nskf = StratifiedKFold(\n    n_splits=n_splits,\n    shuffle=True,\n    random_state=42\n)\n\noof_preds = np.zeros(\n    (len(X), 9)\n)\nmodels = [None] * n_splits\nscalers = [None] * n_splits\n\ndef train_single_fold(\n    fold,\n    train_idx,\n    val_idx\n):\n    X_train = X[train_idx]\n    X_val = X[val_idx]\n    y_train = y[train_idx]\n    y_val = y[val_idx]\n\n    scaler = MinMaxScaler()\n    X_train_scaled = scaler.fit_transform(\n        X_train\n    )\n    X_val_scaled = scaler.transform(\n        X_val\n    )\n\n    gpu_id = fold % 2\n\n    model = xgb.XGBClassifier(\n        n_estimators=1500,\n        learning_rate=0.03,\n        max_depth=8,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        objective='multi:softprob',\n        tree_method='hist',\n        device=f'cuda:{gpu_id}',\n        eval_metric='mlogloss',\n        early_stopping_rounds=100\n    )\n\n    model.fit(\n        X_train_scaled,\n        y_train,\n        eval_set=[\n            (X_val_scaled, y_val)\n        ],\n        verbose=False\n    )\n\n    val_preds = model.predict_proba(\n        X_val_scaled\n    )\n    fold_loss = log_loss(\n        y_val,\n        val_preds\n    )\n\n    print(\n        f\"Fold {fold + 1} \"\n        f\"(GPU {gpu_id}) completed. \"\n        f\"Log loss: {fold_loss:.4f}\"\n    )\n\n    return (\n        fold,\n        val_idx,\n        val_preds,\n        model,\n        scaler\n    )\n\nprint(\n    \"Training five folds in parallel \"\n    \"across two T4 GPUs...\"\n)\n\nresults = Parallel(\n    n_jobs=2,\n    backend='threading'\n)(\n    delayed(train_single_fold)(\n        fold,\n        train_idx,\n        val_idx\n    )\n    for fold, (\n        train_idx,\n        val_idx\n    ) in enumerate(\n        skf.split(X, y)\n    )\n)\n\nfor (\n    fold,\n    val_idx,\n    val_preds,\n    model,\n    scaler\n) in results:\n    oof_preds[val_idx] = val_preds\n    models[fold] = model\n    scalers[fold] = scaler\n\nprint(\"=\" * 60)\nprint(\n    \"Overall out-of-fold log loss: \"\n    f\"{log_loss(y, oof_preds):.4f}\"\n)\nprint(\"=\" * 60)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:47:19.156677Z","iopub.execute_input":"2026-04-21T16:47:19.157305Z","iopub.status.idle":"2026-04-21T16:50:39.703975Z","shell.execute_reply.started":"2026-04-21T16:47:19.157278Z","shell.execute_reply":"2026-04-21T16:50:39.703215Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Out-of-Fold Evaluation\n","metadata":{}},{"cell_type":"code","source":"overall_oof_log_loss = log_loss(\n    y,\n    oof_preds\n)\ny_pred_classes = np.argmax(\n    oof_preds,\n    axis=1\n)\n\nprint(\n    \"Overall out-of-fold log loss: \"\n    f\"{overall_oof_log_loss:.12f}\"\n)\nprint(\n    classification_report(\n        y,\n        y_pred_classes,\n        target_names=[\n            f'Class {index + 1}'\n            for index in range(9)\n        ]\n    )\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:50:52.059406Z","iopub.execute_input":"2026-04-21T16:50:52.060306Z","iopub.status.idle":"2026-04-21T16:50:52.079365Z","shell.execute_reply.started":"2026-04-21T16:50:52.060266Z","shell.execute_reply":"2026-04-21T16:50:52.078608Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Fold Artifact Export\n","metadata":{}},{"cell_type":"code","source":"for fold, (\n    model,\n    scaler\n) in enumerate(\n    zip(models, scalers)\n):\n    model.save_model(\n        f'xgboost_fold_{fold}.json'\n    )\n    joblib.dump(\n        scaler,\n        f'scaler_fold_{fold}.pkl'\n    )\n\nprint(\n    \"Saved all five XGBoost fold models \"\n    \"and fold-specific scalers.\"\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:51:00.727621Z","iopub.execute_input":"2026-04-21T16:51:00.727950Z","iopub.status.idle":"2026-04-21T16:51:01.330992Z","shell.execute_reply.started":"2026-04-21T16:51:00.727923Z","shell.execute_reply":"2026-04-21T16:51:01.330365Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Test Inference and Submission Generation\n","metadata":{}},{"cell_type":"code","source":"print(\n    \"Generating predictions for the test set...\"\n)\n\ndf_test = pd.read_csv(\n    TEST_FEATURES_CSV\n).fillna(0)\ntest_ids = df_test['ID'].values\n\nmissing_cols = list(\n    set(feature_columns)\n    - set(df_test.columns)\n)\n\nif missing_cols:\n    missing_df = pd.DataFrame(\n        0,\n        index=df_test.index,\n        columns=missing_cols\n    )\n    df_test = pd.concat(\n        [df_test, missing_df],\n        axis=1\n    )\n\nX_test_raw = df_test[\n    feature_columns\n].values\n\ntest_preds = np.zeros(\n    (len(X_test_raw), 9)\n)\n\nfor fold, model in enumerate(models):\n    X_test_scaled = scalers[\n        fold\n    ].transform(X_test_raw)\n\n    model.set_params(\n        device='cpu'\n    )\n    test_preds += (\n        model.predict_proba(\n            X_test_scaled\n        )\n        / n_splits\n    )\n\ntest_preds = test_preds / np.sum(\n    test_preds,\n    axis=1,\n    keepdims=True\n)\n\ncolumns = [\n    f'Prediction{index}'\n    for index in range(1, 10)\n]\nsubmission_df = pd.DataFrame(\n    test_preds,\n    columns=columns\n)\nsubmission_df.insert(\n    0,\n    'Id',\n    test_ids\n)\n\nsubmission_file = (\n    'submission_xgboost_cv_optimized.csv'\n)\nsubmission_df.to_csv(\n    submission_file,\n    index=False\n)\n\nprint(\n    f\"Submission file created: \"\n    f\"{submission_file}\"\n)\ndisplay(submission_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-21T16:51:04.921909Z","iopub.execute_input":"2026-04-21T16:51:04.922229Z","iopub.status.idle":"2026-04-21T16:51:14.263297Z","shell.execute_reply.started":"2026-04-21T16:51:04.922200Z","shell.execute_reply":"2026-04-21T16:51:14.262399Z"}},"outputs":[],"execution_count":null}]}