{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4117,"databundleVersionId":46665,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 1. Imports & Settings\nimport os                             # for filesystem operations\nimport subprocess                     # to call 7z for listing/extracting\nimport numpy as np                    # numerical arrays\nimport pandas as pd                   # DataFrame handling\nfrom tqdm import tqdm                 # progress bars\nimport matplotlib.pyplot as plt       # plotting\nimport seaborn as sns                 # nicer plots\n\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import (\n    confusion_matrix,\n    classification_report,\n    precision_score,\n    recall_score,\n    f1_score,\n    accuracy_score\n)\n\n# fix random seed for reproducibility\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\nsns.set_style(\"whitegrid\")\n\n\n# 2. Locate train.7z and trainLabels.csv under /kaggle/input\nbase_input = \"/kaggle/input\"\nzip_dir = None\nfor d in os.listdir(base_input):\n    cand = os.path.join(base_input, d)\n    # check if this folder contains both files\n    if os.path.isdir(cand) and \\\n       \"train.7z\" in os.listdir(cand) and \\\n       \"trainLabels.csv\" in os.listdir(cand):\n        zip_dir = cand\n        break\n\nif zip_dir is None:\n    raise FileNotFoundError(\"Couldn't find train.7z and trainLabels.csv in /kaggle/input\")\n\ntrain_zip   = os.path.join(zip_dir, \"train.7z\")\nlabels_path = os.path.join(zip_dir, \"trainLabels.csv\")\n\n\n# 3. Load labels and build binary target\nlabels_df = pd.read_csv(labels_path)                 # read CSV with Id and Class\nlabels_df[\"Id\"]        = labels_df[\"Id\"].astype(str) # ensure Id is string\nlabels_df[\"Malicious\"] = (labels_df[\"Class\"] > 0).astype(int)  # class>0 → malicious=1\nlabel_ids = set(labels_df[\"Id\"])                     # set for fast lookup\n\n\n# 4. List .bytes entries in the 7z archive (without extracting all)\nlist_output = subprocess.check_output(\n    [\"7z\", \"l\", \"-ba\", \"-slt\", train_zip],  # list all with technical detail\n    stderr=subprocess.DEVNULL\n).decode(\"utf-8\", errors=\"ignore\")\n\nbytes_entries = []\nfor line in list_output.splitlines():\n    # look for lines like \"Path = <filename.bytes>\"\n    if line.startswith(\"Path = \") and line.strip().endswith(\".bytes\"):\n        path = line.split(\" = \", 1)[1]\n        bytes_entries.append(path)\n\nif not bytes_entries:\n    raise RuntimeError(\"No .bytes entries found in train.7z\")\n\nprint(f\"Archive contains {len(bytes_entries)} .bytes files\")\n\n\n# 5. Define feature‐extraction from a raw .bytes file stream\ndef extract_byte_features_from_stream(stream: bytes):\n    \"\"\"\n    Input: raw bytes of a .bytes file\n    Output: (256‐dim byte-value histogram, total line count)\n    Skips '??' tokens which indicate unknown bytes.\n    \"\"\"\n    hist = np.zeros(256, dtype=int)\n    line_count = 0\n\n    for raw_line in stream.split(b'\\n'):\n        if not raw_line:\n            continue\n        line_count += 1\n        # drop the address token; remaining tokens are hex bytes\n        parts = raw_line.split(b' ')[1:]\n        for tok in parts:\n            if tok != b\"??\":\n                try:\n                    hist[int(tok, 16)] += 1\n                except ValueError:\n                    pass\n    return hist, line_count\n\n\n# 6. Stream‐extract features for each labeled sample\nbyte_cols   = [f\"byte_{i:02X}\" for i in range(256)]  # column names 00–FF\nfeatures    = []   # list of histograms\nline_counts = []   # list of line counts\nids         = []   # list of file IDs\n\nfor member in tqdm(bytes_entries, desc=\"Streaming & extracting\"):\n    fid = os.path.splitext(os.path.basename(member))[0]\n    if fid not in label_ids:\n        continue  # skip unlabeled examples\n\n    # extract single .bytes file to stdout\n    proc = subprocess.run(\n        [\"7z\", \"x\", train_zip, member, \"-so\"],\n        stdout=subprocess.PIPE,\n        stderr=subprocess.DEVNULL\n    )\n    content = proc.stdout  # raw bytes data\n\n    hist, lc = extract_byte_features_from_stream(content)\n    features.append(hist)\n    line_counts.append(lc)\n    ids.append(fid)\n\nif not ids:\n    raise RuntimeError(\"No samples extracted. Check IDs vs archive filenames.\")\n\nprint(f\"Extracted features for {len(ids)} samples\")\n\n\n# 7. Build DataFrame & merge with labels\nX_df = pd.DataFrame(features, columns=byte_cols)  # 256 columns\nX_df[\"LineCount\"] = line_counts                  # add line-count feature\nX_df[\"Id\"]        = ids                          # add file ID\n\n# merge to keep only labeled rows and attach the target\ndata = X_df.merge(labels_df[[\"Id\",\"Malicious\"]], on=\"Id\", how=\"inner\")\nprint(\"Final data shape:\", data.shape)\n\n\n# 8. Prepare feature matrix X and target y\nX = data[byte_cols + [\"LineCount\"]].values   # numpy array of features\ny = data[\"Malicious\"].values                 # binary target array\n\n\n# 9. Train/test split (stratified)\nX_train, X_test, y_train, y_test = train_test_split(\n    X, y, test_size=0.2, stratify=y, random_state=RANDOM_STATE\n)\n\n\n# 10. Feature scaling (standardization)\nscaler  = StandardScaler()\nX_train = scaler.fit_transform(X_train)  # fit on train\nX_test  = scaler.transform(X_test)       # apply same transform to test\n\n\n# 11. Define models to compare\nmodels = {\n    \"LogisticRegression\": LogisticRegression(max_iter=1000, random_state=RANDOM_STATE),\n    \"RandomForest\":       RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=RANDOM_STATE)\n}\n\n\n# 12. Cross-validation to estimate F1 on train set\nprint(\"\\nCross-validation F1 scores:\")\nfor name, model in models.items():\n    scores = cross_val_score(model, X_train, y_train, cv=5, scoring=\"f1\", n_jobs=-1)\n    print(f\"  {name}: {scores.mean():.3f} ± {scores.std():.3f}\")\n\n\n# 13. Fit each model & evaluate on the held-out test set\nresults = {}\nfor name, model in models.items():\n    model.fit(X_train, y_train)               # train\n    y_pred = model.predict(X_test)            # predict\n\n    print(f\"\\n=== {name} ===\")\n    # confusion matrix\n    cm = confusion_matrix(y_test, y_pred)\n    sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n    plt.title(f\"{name} Confusion Matrix\")\n    plt.xlabel(\"Predicted\"); plt.ylabel(\"Actual\")\n    plt.show()\n\n    # detailed classification report\n    print(classification_report(y_test, y_pred, target_names=[\"Benign\",\"Malicious\"]))\n\n    # store metrics\n    results[name] = {\n        \"accuracy\":  accuracy_score(y_test, y_pred),\n        \"precision\": precision_score(y_test, y_pred),\n        \"recall\":    recall_score(y_test, y_pred),\n        \"f1\":        f1_score(y_test, y_pred)\n    }\n\n\n# 14. Compare model performance in a table\nres_df = pd.DataFrame(results).T\nprint(\"\\nModel comparison:\\n\", res_df)\n\n\n# 15. Plot top‐10 feature importances for each model\n# RandomForest feature importances\nrf_imp = models[\"RandomForest\"].feature_importances_\ntop_rf = np.argsort(rf_imp)[-10:][::-1]\nplt.figure(figsize=(6,4))\nsns.barplot(x=rf_imp[top_rf], y=np.array(byte_cols+[\"LineCount\"])[top_rf])\nplt.title(\"Top 10 RandomForest Features\")\nplt.tight_layout()\nplt.show()\n\n# LogisticRegression coefficients (absolute)\nlr_coef = models[\"LogisticRegression\"].coef_[0]\ntop_lr  = np.argsort(np.abs(lr_coef))[-10:][::-1]\nplt.figure(figsize=(6,4))\nsns.barplot(x=lr_coef[top_lr], y=np.array(byte_cols+[\"LineCount\"])[top_lr])\nplt.title(\"Top 10 LogisticRegression Coefficients\")\nplt.tight_layout()\nplt.show()\n\n\n\n\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-31T00:40:57.950152Z","iopub.execute_input":"2025-07-31T00:40:57.950552Z","execution_failed":"2025-07-31T00:41:01.212Z"}},"outputs":[],"execution_count":null}]}