{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"databundleVersionId":46665,"sourceId":4117,"sourceType":"competition"}],"dockerImageVersionId":31328,"isGpuEnabled":false,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"papermill":{"default_parameters":{},"duration":7262.035758,"end_time":"2026-04-05T15:40:10.481083+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-04-05T13:39:08.445325+00:00","version":"2.7.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"3433861c-f17e-472c-ac25-f52e8e66d269","cell_type":"markdown","source":"# Experiment 00: Binary Grayscale Images with a Frozen ResNet50\n\nThis notebook establishes an initial transfer-learning baseline for the\nMicrosoft Malware Classification Challenge.\n\n| Component | Configuration |\n|---|---|\n| Input | First 3,000 `.bytes` files extracted from `train.7z` |\n| Representation | 224 × 224 grayscale image replicated across three channels |\n| Backbone | ImageNet-pretrained ResNet50 |\n| Backbone training | Frozen |\n| Data split | Stratified 80% training / 20% validation |\n| Batch size | 16 |\n| Training duration | 10 epochs |\n| Output classes | 9 malware families |\n","metadata":{}},{"id":"aaa6c62b-c12f-4589-8d5a-39a150958678","cell_type":"markdown","source":"## 1. Environment Setup and Data Extraction\n","metadata":{}},{"id":"f0fa3287","cell_type":"code","source":"!apt-get install -y p7zip-full\n!7z x /kaggle/input/competitions/malware-classification/dataSample.7z -o/kaggle/working/sample_data/","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:39:12.846011Z","iopub.status.busy":"2026-04-05T13:39:12.845691Z","iopub.status.idle":"2026-04-05T13:39:19.192791Z","shell.execute_reply":"2026-04-05T13:39:19.191831Z"},"papermill":{"duration":6.354462,"end_time":"2026-04-05T13:39:19.195170+00:00","exception":false,"start_time":"2026-04-05T13:39:12.840708+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"1b274222","cell_type":"code","source":"!7z l /kaggle/input/competitions/malware-classification/train.7z train/*.bytes | grep \"train/.*\\.bytes\" | awk '{print $NF}' | head -n 3000 > files_to_extract.txt\n!7z e /kaggle/input/competitions/malware-classification/train.7z -o/kaggle/working/train_bytes/ @files_to_extract.txt -y","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:39:19.204925Z","iopub.status.busy":"2026-04-05T13:39:19.204600Z","iopub.status.idle":"2026-04-05T13:42:44.279858Z","shell.execute_reply":"2026-04-05T13:42:44.277781Z"},"papermill":{"duration":205.08359,"end_time":"2026-04-05T13:42:44.283083+00:00","exception":false,"start_time":"2026-04-05T13:39:19.199493+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"171acceb-b779-44ae-bc3a-08f94bf81a18","cell_type":"markdown","source":"## 2. Frozen ResNet50 Backbone\n","metadata":{}},{"id":"a6039a65","cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.applications import ResNet50\n\nbase_model = ResNet50(\n    weights='imagenet',\n    include_top=False,\n    input_shape=(224, 224, 3)\n)\nbase_model.trainable = False","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2026-04-05T13:42:44.519514Z","iopub.status.busy":"2026-04-05T13:42:44.518594Z","iopub.status.idle":"2026-04-05T13:43:42.517804Z","shell.execute_reply":"2026-04-05T13:43:42.515907Z"},"papermill":{"duration":58.125831,"end_time":"2026-04-05T13:43:42.520757+00:00","exception":false,"start_time":"2026-04-05T13:42:44.394926+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8b6accb3-84ce-4136-a004-070a8a3731b8","cell_type":"markdown","source":"## 3. Dataset Assembly and Inspection\n","metadata":{}},{"id":"cb496bb3","cell_type":"code","source":"import glob\nimport os\n\nimport pandas as pd\n\ncsv_path = '/kaggle/input/competitions/malware-classification/trainLabels.csv'\ndf = pd.read_csv(csv_path)\n\nbytes_files = glob.glob('/kaggle/working/train_bytes/*.bytes')\nprint(f\"Extracted .bytes files: {len(bytes_files):,}\")\n\nif len(bytes_files) > 0:\n    data = []\n    for path in bytes_files:\n        file_id = os.path.basename(path).replace('.bytes', '')\n        data.append({'Id': file_id, 'filepath': path})\n\n    df_files = pd.DataFrame(data)\n    df_final = pd.merge(df_files, df, on='Id', how='inner')\n\n    print(f\"Successfully matched labeled files: {len(df_final):,}\")\n    display(df_final.head())\nelse:\n    print(\n        \"No .bytes files were found. \"\n        \"Verify that the training archive was extracted successfully.\"\n    )","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:42.600757Z","iopub.status.busy":"2026-04-05T13:43:42.600004Z","iopub.status.idle":"2026-04-05T13:43:42.706917Z","shell.execute_reply":"2026-04-05T13:43:42.705446Z"},"papermill":{"duration":0.150004,"end_time":"2026-04-05T13:43:42.709689+00:00","exception":false,"start_time":"2026-04-05T13:43:42.559685+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0874bc5a","cell_type":"code","source":"import pandas as pd\n\nlabels_path = '/kaggle/input/competitions/malware-classification/trainLabels.csv'\ndf_labels = pd.read_csv(labels_path)\n\nprint(\"Training label preview:\")\ndisplay(df_labels.head())\n\ndf_final = pd.merge(df_files, df_labels, on='Id', how='inner')\nprint(f\"Files retained after label matching: {len(df_final):,}\")","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:42.789023Z","iopub.status.busy":"2026-04-05T13:43:42.788658Z","iopub.status.idle":"2026-04-05T13:43:42.813002Z","shell.execute_reply":"2026-04-05T13:43:42.811019Z"},"papermill":{"duration":0.066844,"end_time":"2026-04-05T13:43:42.815533+00:00","exception":false,"start_time":"2026-04-05T13:43:42.748689+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"601103d0","cell_type":"code","source":"import glob\nimport os\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\n\nfiles = glob.glob('/kaggle/working/train_bytes/*.bytes')\ndata = [\n    {'Id': os.path.basename(file).replace('.bytes', ''), 'filepath': file}\n    for file in files\n]\ndf_files = pd.DataFrame(data)\ndf_final = pd.merge(df_files, df_labels, on='Id', how='inner')\n\nplt.figure(figsize=(10, 5))\nsns.countplot(x='Class', data=df_final)\nplt.title('Malware Family Distribution in the Extracted Training Subset')\nplt.show()\n\nprint(f\"Total labeled files in the working subset: {len(df_final):,}\")","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:42.892808Z","iopub.status.busy":"2026-04-05T13:43:42.892456Z","iopub.status.idle":"2026-04-05T13:43:43.876385Z","shell.execute_reply":"2026-04-05T13:43:43.874944Z"},"papermill":{"duration":1.025401,"end_time":"2026-04-05T13:43:43.878493+00:00","exception":false,"start_time":"2026-04-05T13:43:42.853092+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"bd700e49-b85d-41a5-8871-99c2d21dcf95","cell_type":"markdown","source":"## 4. Class-Imbalance Handling\n","metadata":{}},{"id":"e8dfffc0","cell_type":"code","source":"import numpy as np\nfrom sklearn.utils import class_weight\n\ny_train_labels = df_final['Class'].values - 1\n\nclass_weights = class_weight.compute_class_weight(\n    class_weight='balanced',\n    classes=np.unique(y_train_labels),\n    y=y_train_labels\n)\nclass_weight_dict = dict(enumerate(class_weights))\n\nprint(\"Class weights (higher values indicate underrepresented classes):\")\nprint(class_weight_dict)","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:43.952729Z","iopub.status.busy":"2026-04-05T13:43:43.951890Z","iopub.status.idle":"2026-04-05T13:43:44.013344Z","shell.execute_reply":"2026-04-05T13:43:44.011682Z"},"papermill":{"duration":0.100817,"end_time":"2026-04-05T13:43:44.015307+00:00","exception":false,"start_time":"2026-04-05T13:43:43.914490+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7325a008-e486-4823-8005-b2300e7ee03d","cell_type":"markdown","source":"## 5. Binary-to-Image Transformation\n","metadata":{}},{"id":"191bd6d0","cell_type":"code","source":"import cv2\nimport numpy as np\n\ndef bytes_to_resnet_image(filepath):\n    hex_list = []\n\n    with open(filepath, 'r') as file:\n        for line in file:\n            parts = line.strip().split()\n\n            for hex_code in parts[1:]:\n                if hex_code != '??':\n                    hex_list.append(int(hex_code, 16))\n                else:\n                    hex_list.append(0)\n\n    img_array = np.array(hex_list, dtype=np.uint8)\n    side = int(np.sqrt(len(img_array)))\n    img_matrix = img_array[:side * side].reshape((side, side))\n    img_resized = cv2.resize(\n        img_matrix,\n        (224, 224),\n        interpolation=cv2.INTER_CUBIC\n    )\n    img_rgb = np.stack((img_resized,) * 3, axis=-1)\n\n    return img_rgb\n","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:44.092572Z","iopub.status.busy":"2026-04-05T13:43:44.091373Z","iopub.status.idle":"2026-04-05T13:43:44.099552Z","shell.execute_reply":"2026-04-05T13:43:44.098561Z"},"papermill":{"duration":0.048518,"end_time":"2026-04-05T13:43:44.101984+00:00","exception":false,"start_time":"2026-04-05T13:43:44.053466+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"e3cfa3ef-e3bf-49d3-a5de-3a1898c35f6f","cell_type":"markdown","source":"## 6. Data Generator\n","metadata":{}},{"id":"08c2fd93","cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\n\nclass MalwareDataGenerator(tf.keras.utils.Sequence):\n    def __init__(\n        self,\n        filepaths,\n        labels,\n        batch_size=32,\n        target_size=(224, 224)\n    ):\n        self.filepaths = filepaths\n        self.labels = labels\n        self.batch_size = batch_size\n        self.target_size = target_size\n        self.indices = np.arange(len(self.filepaths))\n\n    def __len__(self):\n        return int(np.ceil(len(self.filepaths) / float(self.batch_size)))\n\n    def __getitem__(self, index):\n        batch_indices = self.indices[\n            index * self.batch_size:(index + 1) * self.batch_size\n        ]\n        X = []\n        y = []\n\n        for i in batch_indices:\n            img = bytes_to_resnet_image(self.filepaths[i])\n            X.append(img)\n            y.append(\n                tf.keras.utils.to_categorical(\n                    self.labels[i],\n                    num_classes=9\n                )\n            )\n\n        return np.array(X), np.array(y)\n","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:44.180482Z","iopub.status.busy":"2026-04-05T13:43:44.179150Z","iopub.status.idle":"2026-04-05T13:43:44.186671Z","shell.execute_reply":"2026-04-05T13:43:44.185719Z"},"papermill":{"duration":0.048616,"end_time":"2026-04-05T13:43:44.188959+00:00","exception":false,"start_time":"2026-04-05T13:43:44.140343+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"efb1aad3-3afc-4628-895a-34c6f69ea1f1","cell_type":"markdown","source":"## 7. Model Training\n","metadata":{}},{"id":"c6ea5e0e","cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.applications import ResNet50\n\nX_paths = df_final['filepath'].values\ny_labels = df_final['Class'].values - 1\n\nX_train, X_val, y_train, y_val = train_test_split(\n    X_paths,\n    y_labels,\n    test_size=0.2,\n    random_state=42,\n    stratify=y_labels\n)\n\ntrain_gen = MalwareDataGenerator(X_train, y_train, batch_size=16)\nval_gen = MalwareDataGenerator(X_val, y_val, batch_size=16)\n\nprint(\"Loading ImageNet-pretrained ResNet50 weights...\")\nbase_model = ResNet50(\n    weights='imagenet',\n    include_top=False,\n    input_shape=(224, 224, 3)\n)\nbase_model.trainable = False\n\nmodel = models.Sequential([\n    base_model,\n    layers.GlobalAveragePooling2D(),\n    layers.Dense(256, activation='relu'),\n    layers.Dropout(0.5),\n    layers.Dense(9, activation='softmax')\n])\n\nmodel.compile(\n    optimizer='adam',\n    loss='categorical_crossentropy',\n    metrics=['accuracy']\n)\n\nprint(\"Starting model training...\")\nhistory = model.fit(\n    train_gen,\n    validation_data=val_gen,\n    epochs=10,\n    class_weight=class_weight_dict,\n    verbose=1\n)","metadata":{"execution":{"iopub.execute_input":"2026-04-05T13:43:44.265510Z","iopub.status.busy":"2026-04-05T13:43:44.265091Z","iopub.status.idle":"2026-04-05T15:40:03.602052Z","shell.execute_reply":"2026-04-05T15:40:03.594107Z"},"papermill":{"duration":6979.41416,"end_time":"2026-04-05T15:40:03.640197+00:00","exception":false,"start_time":"2026-04-05T13:43:44.226037+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"18400c84-eb91-4db3-b399-360ff4290873","cell_type":"markdown","source":"## 8. Evaluation\n","metadata":{}},{"id":"9a5765e4","cell_type":"code","source":"print(\"Final training metrics\")\nprint(f\"Training accuracy: {history.history['accuracy'][-1]:.4f}\")\nprint(f\"Validation accuracy: {history.history['val_accuracy'][-1]:.4f}\")\nprint(f\"Training loss: {history.history['loss'][-1]:.4f}\")\nprint(f\"Validation loss: {history.history['val_loss'][-1]:.4f}\")\n","metadata":{"execution":{"iopub.execute_input":"2026-04-05T15:40:04.132488Z","iopub.status.busy":"2026-04-05T15:40:04.121113Z","iopub.status.idle":"2026-04-05T15:40:04.160955Z","shell.execute_reply":"2026-04-05T15:40:04.159340Z"},"papermill":{"duration":0.16928,"end_time":"2026-04-05T15:40:04.163310+00:00","exception":false,"start_time":"2026-04-05T15:40:03.994030+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"7f80d6b9","cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(10, 4))\n\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Training')\nplt.plot(history.history['val_accuracy'], label='Validation')\nplt.title('Accuracy')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], label='Training')\nplt.plot(history.history['val_loss'], label='Validation')\nplt.title('Loss')\nplt.legend()\n\nplt.show()\n","metadata":{"execution":{"iopub.execute_input":"2026-04-05T15:40:04.380458Z","iopub.status.busy":"2026-04-05T15:40:04.379976Z","iopub.status.idle":"2026-04-05T15:40:05.046905Z","shell.execute_reply":"2026-04-05T15:40:05.044496Z"},"papermill":{"duration":0.786288,"end_time":"2026-04-05T15:40:05.049857+00:00","exception":false,"start_time":"2026-04-05T15:40:04.263569+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"f2957973-6a4f-49f3-a765-98337cad2e5a","cell_type":"markdown","source":"## 9. Model and Training-History Export\n","metadata":{}},{"id":"3faf42cf","cell_type":"code","source":"import pickle\n\nmodel.save('malware_resnet50_final.keras')\n\nwith open('train_history.pkl', 'wb') as file:\n    pickle.dump(history.history, file)\n","metadata":{"execution":{"iopub.execute_input":"2026-04-05T15:40:05.270829Z","iopub.status.busy":"2026-04-05T15:40:05.270383Z","iopub.status.idle":"2026-04-05T15:40:07.629559Z","shell.execute_reply":"2026-04-05T15:40:07.627296Z"},"papermill":{"duration":2.472444,"end_time":"2026-04-05T15:40:07.632865+00:00","exception":false,"start_time":"2026-04-05T15:40:05.160421+00:00","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}