{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4117,"databundleVersionId":46665,"sourceType":"competition"},{"sourceId":9148677,"sourceType":"datasetVersion","datasetId":5526194}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-11T09:21:57.451560Z","iopub.execute_input":"2024-08-11T09:21:57.452025Z","iopub.status.idle":"2024-08-11T09:21:57.462501Z","shell.execute_reply.started":"2024-08-11T09:21:57.451979Z","shell.execute_reply":"2024-08-11T09:21:57.461245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1.1 Problem\n# We are provided with a set of known malware files in train.7z and test.7z zip files. Each set representing a mix of Nine different malware families. Class labels of these malware files are provided in the separate trainLabels.csv CSV file.\n\n# Each malware file in the train and test set has:\n\n# Id: A 20 character hash value uniquely identifying the file as its file-name. Refer zip files.\n# Class: An integer representing one of 9 family names to which the malware may belong. Refer trainLabels.csv CSV file.\n# Types of Malwares:\n\n# Ramnit\n# Lollipop\n# Kelihos_ver3\n# Vundo\n# Simda\n# Tracur\n# Kelihos_ver1\n# Obfuscator.ACY\n# Gatak\n# For each file, the raw data contains the hexadecimal representation of the file's binary content, without the PE header (to ensure sterility). You are also provided a metadata manifest (files with .asm extension), which is a log containing various metadata information extracted from the binary, such as function calls, strings, etc. This was generated using the IDA disassembler tool. Our task is to develop the best mechanism for classifying files in the test set into their respective family affiliations.\n# 1.2 Solution\n# Randomly equal number of malware files from each class (except Simda).\n# Extract hexadecimal strings (of length 2) from the .bytes file.\n# Build Bag-of-Words using scikit-learn CountVectorizer.\n# Build a model using XGBoost XGBClassifier.\n# Note:\n\n# We can improve the solution by using the metadata manifest (files with .asm extension).\n# We can further improve the solution by more using advanced featurization techniques.\n# 1.3 Objective\n# Objective of this kernel is to:\n\n# Pick a balanced sample from the original dataset.\n# Since the dataset is insanely huge, improve the solution by parallelizing computationally intensive tasks and file operations","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:21:58.944373Z","iopub.execute_input":"2024-08-11T09:21:58.945597Z","iopub.status.idle":"2024-08-11T09:21:58.953911Z","shell.execute_reply.started":"2024-08-11T09:21:58.945537Z","shell.execute_reply":"2024-08-11T09:21:58.952380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.4 Project Folder Structure\n\nFinal project structure:\n```\n├── code\n│    ├── Microsoft Malware Classification (with Multiprocessing).ipynb\n│    ├── HexVectorizer.py\n│    └── submission.csv\n└── input\n     ├── malware-classification\n     │    ├── trainLabels.csv\n     │    ├── train\n     │    │    ├── ...\n     │    │    ├── 0ACDbR5M3ZhBJajygTuf.bytes        (10,868 byte-file)\n     │    │    ├── ...\n     │    └── test\n     │         ├── ...\n     │         ├── ITSUPtCmh7WdJcsYDwQ5.bytes        (10,873 byte-file)\n     │         ├── ...\n     └── microsoft-malware-sample\n          ├── trainLabels_bal.csv\n          ├── train_vec.csv\n          └── test_vec.csv\n```","metadata":{}},{"cell_type":"markdown","source":"## 2. Setup\nAdding Custom Modules","metadata":{}},{"cell_type":"code","source":"%%writefile HexVectorizer.py\n\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nimport os\nimport re\n\n# Byte file properties\nLINE_LEN = 16\nADDR_LEN = 9\n\n\ndef hex_to_str(hex_line):\n    \"\"\"\n    Function to strip \\r, \\n and remove address\n    of length eight character+space from hex line.\n    \"\"\"\n    return hex_line.decode().strip()[ADDR_LEN:]\n\n\ndef remove_non_hex(hex_str):\n    \"\"\"\n    Function to remove non hex characters.\n    \"\"\"\n    # Replace non hex characters with empty string.\n    hex_str = re.sub(r\"[^0-9A-F\\s]+\", \"\", hex_str, flags=re.IGNORECASE)\n    # Replace multiple spaces with single space.\n    hex_str = re.sub(r\"\\s+\", \" \", hex_str)\n\n    return hex_str.strip().lower()\n\n\nclass Vectorizer(CountVectorizer):\n    \"\"\"\n    Convert strings to vectors\n    \"\"\"\n\n    def transform_byte_file(self, ts_file):\n        \"\"\"\n        Function to convert a byte-file into a data-point/row in CSV file.\n        Each data-point will have file-name, file-size & byte-string columns.\n        \"\"\"\n\n        # Open the byte-file for reading.\n        with open(ts_file, \"rb\") as byt_f:\n            # Remove memory address in the beginning of each line and concatenate\n            # all lines in the byte-file into a single string separated by space.\n            byt_str = \" \".join([hex_to_str(line) for line in byt_f.readlines()])\n            byt_str = remove_non_hex(byt_str)\n\n            # Get the byte-file name.\n            f_path, _ = os.path.splitext(byt_f.name)  # Full path, extension.\n            _, f_name = f_path.rsplit(\"/\", 1)  # Relative path, file-name.\n\n            # Get the byte-file size.\n            file_info = os.stat(byt_f.name)\n            f_size = file_info.st_size\n\n            bow = self.transform([byt_str]).toarray()[0].tolist()\n            return [f_name, f_size] + bow\n\n    def get_feature_names(self):\n        \"\"\"\n        Function to return vocabulary as feature names.\n        \"\"\"\n        return self.get_feature_names_out().tolist()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:15.357046Z","iopub.execute_input":"2024-08-11T09:24:15.357482Z","iopub.status.idle":"2024-08-11T09:24:15.366340Z","shell.execute_reply.started":"2024-08-11T09:24:15.357448Z","shell.execute_reply":"2024-08-11T09:24:15.365109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Import and configure libraries","metadata":{}},{"cell_type":"code","source":"# Data manipulation libraries.\nimport numpy as np\nimport pandas as pd\n\n# Data visualization libraries.\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom prettytable import PrettyTable\n\n# Data modeling libraries.\nimport sklearn\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler\nfrom sklearn.manifold import TSNE\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils.class_weight import compute_sample_weight\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import GridSearchCV\n\n# General Imports\nimport os\nimport csv\nimport re\nimport time\nimport math\nfrom tqdm import tqdm\nimport multiprocessing\nfrom multiprocessing import Pool\n\n# Custom modules\nfrom HexVectorizer import Vectorizer\n\n\n# Library versions.\nprint(\"NumPy version:\", np.__version__)\nprint(\"Pandas version:\", pd.__version__)\nprint(\"Matplotlib version:\", matplotlib.__version__)\nprint(\"Seaborn version:\", sns.__version__)\nprint(\"Scikit-learn version:\", sklearn.__version__)\n\n# Configure NumPy.\n# Set `Line width` to Maximum 130 characters in the output, post which it will continue in next line.\nnp.set_printoptions(linewidth=130)\n\n# Configure Pandas.\n# Set display width to maximum 130 characters in the output, post which it will continue in next line.\npd.options.display.width = 130\n# pd.options.display.max_rows = None  # Very dangerous! if dataset is large.\n\n# Configure Seaborn.\nsns.set_style(\"whitegrid\")  # Set white background with grid.\nsns.set_palette(\"deep\")  # Set color palette.\nsns.set_context(\"paper\", font_scale=1.5)  # Set font to scale 1.5 more than normal.","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:19.467739Z","iopub.execute_input":"2024-08-11T09:24:19.468734Z","iopub.status.idle":"2024-08-11T09:24:19.482682Z","shell.execute_reply.started":"2024-08-11T09:24:19.468692Z","shell.execute_reply":"2024-08-11T09:24:19.481308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Constants and helper functions","metadata":{}},{"cell_type":"markdown","source":"### 2.4 Multiprocessing","metadata":{}},{"cell_type":"code","source":"# Multiprocessing on Jupyter Notebook on windows:\n# https://stackoverflow.com/a/47374811/5070460\ndef parallelize(task, data):\n    \"\"\"\n    Function to parallelize `task()` for list of items passed as `data`.\n    \"\"\"\n    # Confirm that the code is under main function.\n    if __name__ == \"__main__\":\n        # Set pool size to number of logical-processors available.\n        POOL_SIZE = multiprocessing.cpu_count()\n        pool = Pool(processes=POOL_SIZE)\n        print(\"Pool size:\", POOL_SIZE)\n\n        outputs = []\n        pbar = tqdm(total=len(data))\n        for output in pool.imap_unordered(task, data):\n            outputs.append(output)\n            pbar.update()\n        pbar.close()\n\n        return outputs","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:40.510426Z","iopub.execute_input":"2024-08-11T09:24:40.510969Z","iopub.status.idle":"2024-08-11T09:24:40.521318Z","shell.execute_reply.started":"2024-08-11T09:24:40.510931Z","shell.execute_reply":"2024-08-11T09:24:40.520009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. EDA","metadata":{}},{"cell_type":"code","source":"TRAIN_DIR = \"../input/malware-classification/train/\"\nLABELS_CSV = \"../input/malware-classification/trainLabels.csv\"\nTEST_DIR = \"../input/malware-classification/test/\"\nPREPS_DIR = \"/kaggle/input/msft-challenge\"\nBYTE_EXT = \".bytes\"\n\nBAL_CL_CSV = os.path.join(PREPS_DIR, \"trainLabels_bal.csv\")\nTR_VEC_CSV = os.path.join(PREPS_DIR, \"train_vec.csv\")\nTS_VEC_CSV = os.path.join(PREPS_DIR, \"test_vec.csv\")\n\n# Byte file properties\nLINE_LEN = 16\nADDR_LEN = 9\n\n\n# Malware class-labels.\nMALWARE_CLS = {\n    1: \"Ramnit\",\n    2: \"Lollipop\",\n    3: \"Kelihos_ver3\",\n    4: \"Vundo\",\n    5: \"Simda\",\n    6: \"Tracur\",\n    7: \"Kelihos_ver1\",\n    8: \"Obfuscator.ACY\",\n    9: \"Gatak\",\n}\nmw_codes = lambda: list(MALWARE_CLS.keys())\nmw_names = lambda: list(MALWARE_CLS.values())\n\n\nSAMPLE_SIZE = 200  # Number of samples taken from each class.\n# Function to pick random sample of size SAMPLE_SIZE from each class.\nget_sample = lambda grp: grp.sample(min(SAMPLE_SIZE, len(grp)))\n\n\ndef get_weights(cls):\n    class_weights = {\n        0: 1,\n        1: 1,\n        2: 1,\n        3: 1,\n        4: 4,\n        5: 1,\n        6: 1,\n        7: 1,\n        8: 1,\n    }\n\n    return [class_weights[cl] for cl in cls]","metadata":{"execution":{"iopub.status.busy":"2024-08-11T08:59:48.450557Z","iopub.execute_input":"2024-08-11T08:59:48.450984Z","iopub.status.idle":"2024-08-11T08:59:48.462857Z","shell.execute_reply.started":"2024-08-11T08:59:48.450953Z","shell.execute_reply":"2024-08-11T08:59:48.461565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 3.1 Byte-file class-labels: _trainLabels.csv_","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:42.155696Z","iopub.execute_input":"2024-08-11T09:24:42.156141Z","iopub.status.idle":"2024-08-11T09:24:42.161050Z","shell.execute_reply.started":"2024-08-11T09:24:42.156107Z","shell.execute_reply":"2024-08-11T09:24:42.159850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class-labels DataFrame.\ncl_df = pd.read_csv(LABELS_CSV)\ncl_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:42.706980Z","iopub.execute_input":"2024-08-11T09:24:42.707391Z","iopub.status.idle":"2024-08-11T09:24:42.734459Z","shell.execute_reply.started":"2024-08-11T09:24:42.707362Z","shell.execute_reply":"2024-08-11T09:24:42.733279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows, cols = cl_df.shape\nprint(f\"There are around {rows} malware files available in the Train dataset.\")\ncls_count = cl_df[\"Class\"].value_counts().sort_index()\ncls_count","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:45.413070Z","iopub.execute_input":"2024-08-11T09:24:45.413469Z","iopub.status.idle":"2024-08-11T09:24:45.426021Z","shell.execute_reply.started":"2024-08-11T09:24:45.413439Z","shell.execute_reply":"2024-08-11T09:24:45.424864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Observations\n\n# Dataset is highly imbalanced.\n# Only 42 byte-files are available of malware class 5 (Simda).","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:48.442038Z","iopub.execute_input":"2024-08-11T09:24:48.442581Z","iopub.status.idle":"2024-08-11T09:24:48.448139Z","shell.execute_reply.started":"2024-08-11T09:24:48.442535Z","shell.execute_reply":"2024-08-11T09:24:48.446719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Balanced subset from Original dataset","metadata":{}},{"cell_type":"code","source":"if os.path.exists(BAL_CL_CSV) and os.stat(BAL_CL_CSV).st_size > 0:\n    cl_df_b = pd.read_csv(BAL_CL_CSV)\nelse:\n    cl_df_b = cl_df.groupby(\"Class\", group_keys=False).apply(get_sample)\n    # Save DataFrame as CSV for future use.\n    cl_df_b.to_csv(BAL_CL_CSV, index=False)\n\nrows, cols = cl_df_b.shape\nprint(f\"Balanced subset will contain data of {rows} byte-files.\")\n\ncls_count = cl_df_b[\"Class\"].value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:50.383773Z","iopub.execute_input":"2024-08-11T09:24:50.384187Z","iopub.status.idle":"2024-08-11T09:24:50.399591Z","shell.execute_reply.started":"2024-08-11T09:24:50.384158Z","shell.execute_reply":"2024-08-11T09:24:50.398119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 6))\n\nplt.pie(x=cls_count, labels=mw_names(), autopct=\"%1.0f%%\")\nplt.title(\"Malware Class-labels\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:50.855752Z","iopub.execute_input":"2024-08-11T09:24:50.856654Z","iopub.status.idle":"2024-08-11T09:24:51.115736Z","shell.execute_reply.started":"2024-08-11T09:24:50.856612Z","shell.execute_reply":"2024-08-11T09:24:51.114124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Data wrangling\n### 4.1 The Problem\n\nData is provided as _.bytes_ and _.asm_ files. We have to convert this raw data to features/vectors before building a classification model. For example a _.bytes_ file contain data as show below. Sample taken from _0ACDbR5M3ZhBJajygTuf.bytes_ file:\n```\n...\n0046F4B0 4E 01 82 01 00 01 AE 01 0E 11 E2 11 0A 10 2A 01\n0046F4C0 60 01 C8 00 E8 10 28 01 04 00 82 00 62 10 E4 01\n0046F4D0 EA 00 CE 01 A6 01 46 11 0C 00 00 00 ?? ?? ?? ??\n0046F4E0 ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ??\n0046F4F0 ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ??\n0046F500 ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ?? ??\n...\n```\n\nWhere,\n\n1. First eight characters are **memory address**.\n2. A hexadecimal representation of the command (byte), always in 2 characters.\n3. Missing values with `??` placeholder.\n\n### 4.2 Introducing `HexVectorizer`\n\nBuild a custom Vectorizer called `HexVectorizer` to:\n\n1. Remove memory-address and `??` placeholder.\n2. Add file-size as a feature.\n3. Replace string with its BoW vector representation.\n\n#### Vocabulary\n\nSince we already know that byte-files contain only hexadecimal characters and hex characters range b/w `0-9` followed by `A-F`, we can build the vocabulary with 256 (16X16) unique words in it.","metadata":{}},{"cell_type":"code","source":"hex_chars = [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\"]\n\nvectorizer = Vectorizer()\nvectorizer.vocabulary = [f\"{i}{j}\" for i in hex_chars for j in hex_chars]","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:24:57.269859Z","iopub.execute_input":"2024-08-11T09:24:57.270298Z","iopub.status.idle":"2024-08-11T09:24:57.277394Z","shell.execute_reply.started":"2024-08-11T09:24:57.270263Z","shell.execute_reply":"2024-08-11T09:24:57.275966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer.vocabulary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.3 Parallelized File Processing\n\nImport `HexVectorizer` module and pass it to `multiprocessing` `Pool` class for processing multiple files in parallel.","metadata":{}},{"cell_type":"code","source":"final_ftrs = [\"Id\", \"BytFSize\"] + vectorizer.get_feature_names()\nfile_paths = [os.path.join(TRAIN_DIR, Id) + BYTE_EXT for Id in cl_df_b[\"Id\"]]\nfile_count = len(file_paths)\n\nprint(f\"Select {file_count} byte-files from unzipped files for processing.\")\n\nif os.path.exists(TR_VEC_CSV) and os.stat(TR_VEC_CSV).st_size > 0:\n    print(\"Sample is already vectorized in:\", TR_VEC_CSV)\n    train_df = pd.read_csv(TR_VEC_CSV)\nelse:\n    print(\"Vectorization of balanced sample from original dataset...\")\n\n    # Parallelized File Processing.\n    tr_ftr_vecs = parallelize(vectorizer.transform_byte_file, file_paths)\n\n    # Convert vectors to DataFrame.\n    train_df = pd.DataFrame(tr_ftr_vecs, columns=final_ftrs)\n\n    def get_class(Id):\n        fltr = cl_df_b[\"Id\"] == Id\n        return cl_df_b.loc[fltr, \"Class\"].item()\n\n    # Append class labels.\n    train_df[\"Class\"] = train_df[\"Id\"].apply(get_class)\n\n    # Save DataFrame as CSV for future use.\n    train_df.to_csv(TR_VEC_CSV, index=False)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:25:04.250641Z","iopub.execute_input":"2024-08-11T09:25:04.251093Z","iopub.status.idle":"2024-08-11T09:25:04.367110Z","shell.execute_reply.started":"2024-08-11T09:25:04.251058Z","shell.execute_reply":"2024-08-11T09:25:04.365839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:31:10.792681Z","iopub.execute_input":"2024-08-11T09:31:10.793104Z","iopub.status.idle":"2024-08-11T09:31:10.819945Z","shell.execute_reply.started":"2024-08-11T09:31:10.793075Z","shell.execute_reply":"2024-08-11T09:31:10.818593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Feature Engineering\n\nThere is nothing much left to engineer any features since non-hex characters and unwanted data are already removed during data wrangling.","metadata":{}},{"cell_type":"code","source":"train_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:34:09.112001Z","iopub.execute_input":"2024-08-11T09:34:09.112438Z","iopub.status.idle":"2024-08-11T09:34:09.141019Z","shell.execute_reply.started":"2024-08-11T09:34:09.112407Z","shell.execute_reply":"2024-08-11T09:34:09.139468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:34:12.924841Z","iopub.execute_input":"2024-08-11T09:34:12.925272Z","iopub.status.idle":"2024-08-11T09:34:12.962743Z","shell.execute_reply.started":"2024-08-11T09:34:12.925240Z","shell.execute_reply":"2024-08-11T09:34:12.961296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows, cols = train_df.shape\nprint(f\"Dataset contains {rows} rows and {cols} columns\")","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:34:18.805625Z","iopub.execute_input":"2024-08-11T09:34:18.806022Z","iopub.status.idle":"2024-08-11T09:34:18.812672Z","shell.execute_reply.started":"2024-08-11T09:34:18.805990Z","shell.execute_reply":"2024-08-11T09:34:18.810938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.1 Validate class-labels","metadata":{}},{"cell_type":"code","source":"for uid, cl in train_df[[\"Id\", \"Class\"]].values:\n    fltr = (cl_df[\"Id\"] == uid) & (cl_df[\"Class\"] == cl)\n    f_id = cl_df.loc[fltr, \"Id\"]\n    if f_id.empty:\n        raise ValueError(f\"Id: {uid} with class: {cl} not found in cl_df.\")\n\nprint(\"All file-ids and class-labels are matching.\")","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:34:26.332725Z","iopub.execute_input":"2024-08-11T09:34:26.333126Z","iopub.status.idle":"2024-08-11T09:34:31.450453Z","shell.execute_reply.started":"2024-08-11T09:34:26.333096Z","shell.execute_reply":"2024-08-11T09:34:31.449326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.2 Feature Scaling","metadata":{}},{"cell_type":"code","source":"X = train_df.drop([\"Id\", \"Class\"], axis=1)\ny = train_df[\"Class\"]\n\n# Centering data-points.\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:34:52.424190Z","iopub.execute_input":"2024-08-11T09:34:52.424623Z","iopub.status.idle":"2024-08-11T09:34:52.460429Z","shell.execute_reply.started":"2024-08-11T09:34:52.424590Z","shell.execute_reply":"2024-08-11T09:34:52.459255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.3 High dimensional data visualization\n#### T-SNE","metadata":{}},{"cell_type":"code","source":"# import warnings\n# warnings.filterwarnings(\"ignore\")\n\ntsne = TSNE(\n    n_components=2,\n    perplexity=10,\n    init=\"random\",\n    learning_rate=\"auto\",\n    verbose=1,\n    random_state=42,\n    n_jobs=-1,\n)\nz = tsne.fit_transform(X_scaled)\nkld = np.round(tsne.kl_divergence_, 4)\n\nprint(\"KL-Divergence:\", kld)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:34:56.951378Z","iopub.execute_input":"2024-08-11T09:34:56.951815Z","iopub.status.idle":"2024-08-11T09:35:05.558418Z","shell.execute_reply.started":"2024-08-11T09:34:56.951781Z","shell.execute_reply":"2024-08-11T09:35:05.557397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame()\n\ndf[\"Class\"] = y.astype(\"category\")\ndf[\"Comp-1\"] = z[:, 0]\ndf[\"Comp-2\"] = z[:, 1]\n\nplt.figure(figsize=(12, 7))\n\nsns.scatterplot(data=df, x=\"Comp-1\", y=\"Comp-2\", hue=\"Class\")\nplt.title(f\"T-SNE projection. KL-Divergence score: {kld}\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:13.472777Z","iopub.execute_input":"2024-08-11T09:35:13.473187Z","iopub.status.idle":"2024-08-11T09:35:14.324411Z","shell.execute_reply.started":"2024-08-11T09:35:13.473157Z","shell.execute_reply":"2024-08-11T09:35:14.323094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 6. Model Training\nle = LabelEncoder()\ny = le.fit_transform(y)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:15.827919Z","iopub.execute_input":"2024-08-11T09:35:15.828337Z","iopub.status.idle":"2024-08-11T09:35:15.834839Z","shell.execute_reply.started":"2024-08-11T09:35:15.828305Z","shell.execute_reply":"2024-08-11T09:35:15.833496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 6.1 Train, Test split","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_cval, y_train, y_cval = train_test_split(\n    X_scaled,\n    y,\n    stratify=y,\n    shuffle=True,\n    test_size=0.3,\n    random_state=42,\n)\n\nprint(\"Train dataset shape:\", X_train.shape)\nprint(\"Cross-Val dataset shape:\", X_cval.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:26.838902Z","iopub.execute_input":"2024-08-11T09:35:26.839306Z","iopub.status.idle":"2024-08-11T09:35:26.853836Z","shell.execute_reply.started":"2024-08-11T09:35:26.839276Z","shell.execute_reply":"2024-08-11T09:35:26.852046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.2 Baseline Model","metadata":{}},{"cell_type":"code","source":"base_m = XGBClassifier(random_state=42, n_jobs=-1)\nbase_m.fit(X_train, y_train, sample_weight=get_weights(y_train))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:30.473458Z","iopub.execute_input":"2024-08-11T09:35:30.473930Z","iopub.status.idle":"2024-08-11T09:35:35.127179Z","shell.execute_reply.started":"2024-08-11T09:35:30.473895Z","shell.execute_reply":"2024-08-11T09:35:35.126032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6.3 Accuracy","metadata":{}},{"cell_type":"code","source":"train_acc = base_m.score(X_train, y_train)\ntest_acc = base_m.score(X_cval, y_cval)\n\nprint(\"Train accuracy:\", round(train_acc, 4))\nprint(\"Test accuracy:\", round(test_acc, 4))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:35.129828Z","iopub.execute_input":"2024-08-11T09:35:35.130494Z","iopub.status.idle":"2024-08-11T09:35:35.152396Z","shell.execute_reply.started":"2024-08-11T09:35:35.130448Z","shell.execute_reply":"2024-08-11T09:35:35.151434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### 6.4 Log Loss","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_probs = base_m.predict_proba(X_cval)\nll = log_loss(y_cval, y_probs)\n\nprint(\"Log loss:\", round(ll, 4))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:37.250647Z","iopub.execute_input":"2024-08-11T09:35:37.251053Z","iopub.status.idle":"2024-08-11T09:35:37.268563Z","shell.execute_reply.started":"2024-08-11T09:35:37.251025Z","shell.execute_reply":"2024-08-11T09:35:37.267442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Hyperparameter Tuning","metadata":{}},{"cell_type":"code","source":"def get_log_loss(kwargs):\n    xgb_clf = XGBClassifier(random_state=42, n_jobs=-1, **kwargs)\n    xgb_clf.fit(X_train, y_train)\n\n    y_train_probs = xgb_clf.predict_proba(X_train)\n    train_ll = log_loss(y_train, y_train_probs)\n\n    y_test_probs = xgb_clf.predict_proba(X_cval)\n    test_ll = log_loss(y_cval, y_test_probs)\n\n    return train_ll, test_ll\n\n\ndef hyperparameter_tuning(**kwargs):\n    train_errs = []\n    test_errs = []\n\n    params = list(kwargs.items())\n    hypr_params = dict(params[0:-1])\n    # Get the last item with range.\n    param_key, range_vals = params[-1]\n\n    for param_val in range_vals:\n        hypr_params[param_key] = param_val\n        train_err, test_err = get_log_loss(hypr_params)\n        train_errs.append(train_err)\n        test_errs.append(test_err)\n\n    plt.figure(figsize=(5, 4))\n\n    sns.lineplot(x=range_vals, y=train_errs, label=\"Train error\")\n    sns.lineplot(x=range_vals, y=test_errs, label=\"Test error\")\n    plt.title(f\"Param: `{param_key}`\")\n    plt.xlabel(\"Value\")\n    plt.ylabel(\"Log Loss\")\n    plt.xticks(range_vals)\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:35:43.405417Z","iopub.execute_input":"2024-08-11T09:35:43.405866Z","iopub.status.idle":"2024-08-11T09:35:43.418199Z","shell.execute_reply.started":"2024-08-11T09:35:43.405833Z","shell.execute_reply":"2024-08-11T09:35:43.416659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 7.2 Tuning\n\nTo decide what range to use for above hyperparameters, plot Train vs Validation (or test) error and pick the sweet spot where:\n1. Test-error is least \n2. Gap b/w Train-error and Train-error is least.","metadata":{}},{"cell_type":"markdown","source":"#### 1. Param: `n_estimators`\n#### 2. Param: `max_depth`\n#### 3. Param: `learning_rate`\n#### 4. Param: `reg_alpha`\n#### 5. Param: `reg_lambda`","metadata":{}},{"cell_type":"code","source":"hyperparameter_tuning(n_estimators=range(1, 70, 10))\nhyperparameter_tuning(n_estimators=40, max_depth=range(1, 10))\nhyperparameter_tuning(max_depth=4, n_estimators=40, learning_rate=np.arange(0.1, 0.55, 0.05))\nhyperparameter_tuning(\n    max_depth=4,\n    n_estimators=40,\n    learning_rate=0.3,\n    reg_alpha=np.arange(0.1, 1, 0.2),\n)\nhyperparameter_tuning(\n    max_depth=3,\n    n_estimators=40,\n    learning_rate=0.3,\n    reg_lambda=np.arange(0.1, 1, 0.2),\n)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:36:05.406684Z","iopub.execute_input":"2024-08-11T09:36:05.407688Z","iopub.status.idle":"2024-08-11T09:37:42.424053Z","shell.execute_reply.started":"2024-08-11T09:36:05.407643Z","shell.execute_reply":"2024-08-11T09:37:42.422781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 7.3 Grid Search CV\n\nPerform Grid Search CV using XGBClassifier model to further fine tune parameters.","metadata":{}},{"cell_type":"code","source":"grid = {\n    \"n_estimators\": [40, 45, 50],\n    \"max_depth\": [3, 4, 5],\n    \"learning_rate\": [0.35, 0.4, 0.45],\n}\n\nstart = time.time()\n\nxgb_clf1 = XGBClassifier(random_state=42, n_jobs=-1)\nxgb_cvm = GridSearchCV(estimator=xgb_clf1, param_grid=grid, n_jobs=-1, cv=5)\nxgb_cvm.fit(X, y, sample_weight=get_weights(y))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:37:42.426153Z","iopub.execute_input":"2024-08-11T09:37:42.426578Z","iopub.status.idle":"2024-08-11T09:40:54.049396Z","shell.execute_reply.started":"2024-08-11T09:37:42.426539Z","shell.execute_reply":"2024-08-11T09:40:54.047981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end = time.time()\nprint(\"Time taken in hyperparameter tuning:\", math.ceil((end - start) / 60), \"mins.\")\n\ntuned_model = xgb_cvm.best_estimator_\n\nprint(\"\\nBest parameters:\")\nprint(xgb_cvm.best_params_)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:40:54.051376Z","iopub.execute_input":"2024-08-11T09:40:54.052329Z","iopub.status.idle":"2024-08-11T09:40:54.059635Z","shell.execute_reply.started":"2024-08-11T09:40:54.052280Z","shell.execute_reply":"2024-08-11T09:40:54.058359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Accuracy:\", round(xgb_cvm.best_score_, 4))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:40:54.062581Z","iopub.execute_input":"2024-08-11T09:40:54.062967Z","iopub.status.idle":"2024-08-11T09:40:54.081871Z","shell.execute_reply.started":"2024-08-11T09:40:54.062935Z","shell.execute_reply":"2024-08-11T09:40:54.080596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = XGBClassifier(**xgb_cvm.best_params_, random_state=42, n_jobs=-1)\nbest_model.fit(X, y, sample_weight=get_weights(y))","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:40:54.083288Z","iopub.execute_input":"2024-08-11T09:40:54.083703Z","iopub.status.idle":"2024-08-11T09:40:56.876350Z","shell.execute_reply.started":"2024-08-11T09:40:54.083672Z","shell.execute_reply":"2024-08-11T09:40:56.875400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Model Testing","metadata":{}},{"cell_type":"code","source":"### 8.1 Feature Engineering\n# Load vectorized Test dataset from a CSV file if present otherwise generate feature vectors from byte file.","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:42:10.216427Z","iopub.execute_input":"2024-08-11T09:42:10.218073Z","iopub.status.idle":"2024-08-11T09:42:10.223365Z","shell.execute_reply.started":"2024-08-11T09:42:10.218017Z","shell.execute_reply":"2024-08-11T09:42:10.221967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(TS_VEC_CSV) and os.stat(TS_VEC_CSV).st_size > 0:\n    print(\"Loading vectors from\", TS_VEC_CSV)\n    test_df = pd.read_csv(TS_VEC_CSV, index_col=\"Id\")\nelse:\n    # Get complete list of byte-files in input directory.\n    _, _, ts_file_names = next(os.walk(TEST_DIR))\n\n    # Generate list for file paths.\n    ts_file_paths = [os.path.join(TEST_DIR, fn) for fn in ts_file_names]\n\n    print(f\"Test dataset contains {len(ts_file_paths)} files.\")\n    print(\"Vectorization of test dataset...\")\n\n    # Parallelized File Processing.\n    ts_ftr_vecs = parallelize(vectorizer.transform_byte_file, ts_file_paths)\n\n    # Convert vectors to DataFrame.\n    test_df = pd.DataFrame(ts_ftr_vecs, columns=final_ftrs)\n    test_df.set_index([\"Id\"], inplace=True)\n\n    # Save DataFrame as CSV for future use.\n    test_df.to_csv(TS_VEC_CSV, index_label=\"Id\")\n\ntest_df.sample(3)","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:42:13.661294Z","iopub.execute_input":"2024-08-11T09:42:13.661768Z","iopub.status.idle":"2024-08-11T09:42:14.241664Z","shell.execute_reply.started":"2024-08-11T09:42:13.661734Z","shell.execute_reply":"2024-08-11T09:42:14.240398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows, cols = test_df.shape\nprint(f\"Test dataset contains {rows} rows and {cols} columns.\")","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:42:14.916667Z","iopub.execute_input":"2024-08-11T09:42:14.917114Z","iopub.status.idle":"2024-08-11T09:42:14.924157Z","shell.execute_reply.started":"2024-08-11T09:42:14.917082Z","shell.execute_reply":"2024-08-11T09:42:14.922840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.2 Predictions","metadata":{}},{"cell_type":"code","source":"predictions = np.round(best_model.predict_proba(test_df).tolist(), 1).tolist()\n\nprint(\"Sample rows from predictions:\")\npredictions[:5]","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:42:18.649279Z","iopub.execute_input":"2024-08-11T09:42:18.649722Z","iopub.status.idle":"2024-08-11T09:42:18.774167Z","shell.execute_reply.started":"2024-08-11T09:42:18.649682Z","shell.execute_reply":"2024-08-11T09:42:18.772949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.column_stack((test_df.index, predictions))\ncol_names = [\"Id\"] + [f\"Prediction{i}\" for i in range(1, 10)]\n\noutput = pd.DataFrame(data, columns=col_names)\noutput.to_csv(\"submission.csv\", index=False)\nprint(\"Your submission was successfully saved!\")","metadata":{"execution":{"iopub.status.busy":"2024-08-11T09:42:20.325137Z","iopub.execute_input":"2024-08-11T09:42:20.325576Z","iopub.status.idle":"2024-08-11T09:42:20.433421Z","shell.execute_reply.started":"2024-08-11T09:42:20.325541Z","shell.execute_reply":"2024-08-11T09:42:20.432091Z"},"trusted":true},"execution_count":null,"outputs":[]}]}