{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [Web3 Phishing Scammer Detection Competition](https://www.kaggle.com/competitions/forta-protect-web3) Benchmark Model: EasyEnsemble with One-Order Neighbors' Features\n\nFeatures and model set-up were inspired by Chen, Weili, et al. “Phishing Scam Detection on Ethereum: Towards Financial Security for Blockchain Ecosystem.”\n\nProceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence, 2020, https://doi.org/10.24963/ijcai.2020/621.","metadata":{}},{"cell_type":"code","source":"from psutil import cpu_count\n\nimport dask.dataframe as dd\nfrom dask.distributed import Client\n\nfrom imblearn.ensemble import EasyEnsembleClassifier\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import average_precision_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBClassifier","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client = Client()\nclient","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get transaction data\ndata_dir = '/kaggle/input/forta-protect-web3'\n\nTX_DTYPES = {'value': 'object'}\nCOLUMNS = ['to_address', 'from_address', 'value', 'block_number']\n\ntrain_dfs = dd.read_csv(f'{data_dir}/eoa_tx_train/eoa_tx_train/*.csv', dtype=TX_DTYPES, usecols=COLUMNS)\ntest_dfs = dd.read_csv(f'{data_dir}/eoa_tx_train/eoa_tx_test/*.csv', dtype=TX_DTYPES, usecols=COLUMNS)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get training ground truth values and ids\ntrain_ids = pd.read_csv(f'{data_dir}/train.csv')\ntest_ids = pd.read_csv(f'{data_dir}/test.csv')","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get one-order neighbor statistics\none_order_dir = f'{data_dir}/one_order/one_order'\n\n# Outgoing transaction value and time related statistics for EOAs’ first-degree neighbors who sent a transaction to the EOAs (aka “from neighbor”)\nfrom_out = pd.read_csv(f'{one_order_dir}/one_order_from_out_stats_train.csv')\nfrom_out_test = pd.read_csv(f'{one_order_dir}/one_order_from_out_stats_test.csv')\n\n# Incoming transaction value and time related statistics for EOAs’ first-degree neighbors who received a transaction from the EOAs (aka “to neighbor”)\nto_in = pd.read_csv(f'{one_order_dir}/one_order_to_in_stats_train.csv')\nto_in_test = pd.read_csv(f'{one_order_dir}/one_order_to_in_stats_test.csv')","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_features(tx_df, labels, from_out, to_in):\n    # Convert value from WEI to ETH\n    tx_df['value_eth'] = ('.' + tx_df['value'].str.zfill(18)).astype(float)\n    incoming = tx_df.merge(labels, how='left', left_on='to_address', right_on='address')\n    incoming = incoming[incoming['address'].notnull()].compute()\n    \n    incoming_agg = incoming.groupby('to_address').agg({'block_number': 'std', \n                                                       'from_address': ['count', 'nunique'],\n                                                       'value_eth': ['min', 'mean']}).reset_index()\n    incoming_agg.columns = [\"_\".join(filter(None, name)) for name in incoming_agg.columns.to_flat_index()]\n    # in_block_number_std: standard deviation of blockNumber of all incoming transactions for an EOA\n    # in_value_min: minimum value of all incoming transactions for an EOA\n    # in_value_mean: mean value of all incoming transactions for an EOA\n    # from_address_count: total incoming transaction count for an EOA\n    # from_address_nunique: total incoming transaction from unique address for an EOA\n    incoming_agg = incoming_agg.rename(columns={'block_number_std': 'in_block_number_std',\n                                                'value_eth_min': 'in_value_min',\n                                                'value_eth_mean': 'in_value_mean'})\n    # from_address_count_unique_ratio: of all incoming transactions, what ratio is unique?\n    incoming_agg['from_address_count_unique_ratio'] = incoming_agg['from_address_nunique'] / incoming_agg['from_address_count']\n    # for std with 1 transaction, fill na with 0\n    # https://stackoverflow.com/questions/32130954/pandas-standard-deviation-returns-nan\n    incoming_agg['in_block_number_std'] = incoming_agg['in_block_number_std'].fillna(0)\n    \n    from_out_agg = incoming[['from_address', 'to_address']].merge(from_out, how='left', left_on='from_address', right_on='address')\n    from_out_agg = from_out_agg[from_out_agg['address'].notnull()]\n    from_out_agg = from_out_agg.drop_duplicates(subset=['from_address', 'to_address'])\n    from_out_agg = from_out_agg.groupby('to_address').agg({'median_value_eth': 'sum',\n                                                           'min_value_eth': 'sum',\n                                                           'std_pop_value_eth': 'min',\n                                                           'block_time_span': 'median',\n                                                           'block_std_pop': 'median'}).reset_index()\n    # from_out_sum_min: sum of from friends' outgoing minimum transaction values\n    # from_out_sum_median: sum of from friends' outgoing median transaction values\n    # from_out_min_std: min of from friends' outgoing std transaction values\n    # from_out_block_timespan_median: median of from friends' outgoing transaction timespans\n    # from_out_block_std_median: median of from friends' outgoing median transaction block number \n    from_out_agg = from_out_agg.rename(columns={'min_value_eth_sum': 'from_out_sum_min', \n                                                'median_value_eth_sum': 'from_out_sum_median', \n                                                'std_pop_value_eth_min': 'from_out_min_std',\n                                                'block_time_span_median': 'from_out_block_timespan_median',\n                                                'block_std_pop_median': 'from_out_block_std_median'})\n    \n    outgoing = tx_df.merge(labels, how='left', left_on='from_address', right_on='address')\n    outgoing = outgoing[outgoing['address'].notnull()].compute()\n    \n    # features similar to incoming_agg but for outgoing transactions\n    outgoing_agg = outgoing.groupby('from_address').agg({'block_number': 'std', \n                                                         'to_address': ['count', 'nunique'],\n                                                         'value_eth': ['min', 'mean']}).reset_index()\n    outgoing_agg.columns = [\"_\".join(filter(None, name)) for name in outgoing_agg.columns.to_flat_index()]\n    outgoing_agg = outgoing_agg.rename(columns={'block_number_std': 'out_block_number_std',\n                                                'value_eth_min': 'out_value_min',\n                                                'value_eth_mean': 'out_value_mean'})\n    outgoing_agg['out_block_number_std'] = outgoing_agg['out_block_number_std'].fillna(0)\n    \n    to_in_agg = outgoing[['from_address', 'to_address']].merge(to_in, how='left', left_on='to_address', right_on='address')\n    to_in_agg = to_in_agg[to_in_agg['address'].notnull()]\n    to_in_agg = to_in_agg.drop_duplicates(subset=['from_address', 'to_address'])\n    to_in_agg = to_in_agg.groupby('from_address').agg({'min_value_eth': ['mean', 'sum'],\n                                                       'median_value_eth': 'sum',\n                                                       'std_pop_value_eth': 'min',\n                                                       'block_time_span': 'median',\n                                                       'block_std_pop': 'median'}).reset_index()\n    to_in_agg.columns = [\"_\".join(filter(None, name)) for name in to_in_agg.columns.to_flat_index()]\n    # to_in_mean_min: mean of to friends' incoming minimum transaction values\n    # to_in_sum_min: sum of to friends' incoming minimum transaction values\n    # to_in_sum_median: sum of to friends' incoming median transaction values\n    # to_in_min_std: min of to friends' incoming std transaction values\n    # to_in_block_timespan_median: median of to friends' incoming transaction timespans\n    # to_in_block_std_median: median of to friends' incoming median transaction block number \n    to_in_agg = to_in_agg.rename(columns={'min_value_eth_mean': 'to_in_mean_min', \n                                          'min_value_eth_sum': 'to_in_sum_min', \n                                          'median_value_eth_sum': 'to_in_sum_median', \n                                          'std_pop_value_eth_min': 'to_in_min_std',\n                                          'block_time_span_median': 'to_in_block_timespan_median',\n                                          'block_std_pop_median': 'to_in_block_std_median'})\n    \n    dt = labels.merge(incoming_agg, how='left', left_on='address', right_on='to_address')\n    dt = dt.merge(outgoing_agg, how='left', left_on='address', right_on='from_address')\n    dt = dt.drop(['to_address', 'from_address'], axis=1)\n    dt = dt.merge(from_out_agg, how='left', left_on='address', right_on='to_address')\n    dt = dt.merge(to_in_agg, how='left', left_on='address', right_on='from_address')\n    dt = dt.drop(['to_address', 'from_address'], axis=1)\n    # total_tx: incoming and outgoing transactions for an EOA\n    dt['total_tx'] = dt['from_address_count'] + dt['to_address_count']\n    dt = dt.fillna(0)\n    \n    return dt","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_feature_and_labels(features, no_labels=False):\n    EXCLUDE_COLS = {'target', 'address', 'Id', 'first_tx_timestamp','last_tx_timestamp'}\n    x_cols = [col for col in features.columns if col not in EXCLUDE_COLS]\n\n    X = features[x_cols] \n    y = None if no_labels else features['target']\n    \n    return X, y","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = generate_features(train_dfs, train_ids, from_out, to_in)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = get_feature_and_labels(train_data)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_SEED = 42\nX_train, X_val, y_train, y_val = train_test_split(X, y, \n                                                  test_size=0.1, \n                                                  random_state=RANDOM_SEED,\n                                                  stratify=y,\n                                                  shuffle=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = EasyEnsembleClassifier(n_estimators=800, \n                             sampling_strategy=0.01, \n                             base_estimator=XGBClassifier(random_state=RANDOM_SEED), \n                             random_state=RANDOM_SEED, \n                             n_jobs=cpu_count()).fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import dump, load\n\ndump(clf, 'sept-2-2022-easy-ensemble-2.joblib') ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val_preds = np.array([class_proba[1] for class_proba in clf.predict_proba(X_val)])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_names = ['normal', 'phishing-scam']\n\nprint(f\"Average Precision Score: {average_precision_score(y_val, y_val_preds)}\")\nprint(classification_report(y_val, y_val_preds >= 0.5, target_names=target_names))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate Submission For Test set\n\ntest_data = generate_features(test_dfs, test_ids, from_out_test, to_in_test)\nX_test, y_test = get_feature_and_labels(test_data, no_labels=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_preds = np.array([class_proba[1] for class_proba in clf.predict_proba(X_test)])\nsubmission = test_data[['Id']].copy()\nsubmission['Predicted'] = y_test_preds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('benchmark_submission.csv', index=None)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\n# Calculate prediction latency\n\nclf.predict_proba(X_test.sample(1))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Areas of Improvement\n\n1. Replace base estimator Xgboost with [lightGBM](https://github.com/microsoft/LightGBM) to reduce prediction latency (See [Which algorithm takes the crown: Light GBM vs XGBOOST?](https://www.analyticsvidhya.com/blog/2017/06/which-algorithm-takes-the-crown-light-gbm-vs-xgboost/)).\n2. As mentioned in the data cleaning section of “Phishing Scam Detection on Ethereum: Towards Financial Security for Blockchain Ecosystem.”, remove Smart Contract transactions from training data. \n\n> 4.2 Data Cleaning - First of all, smart contracts often have complex logic and are not convenient for phishing scams. Furthermore, smart contracts account for very little in the phishing addresses (i.e., 2.6%), and they usually relate to tokens.","metadata":{}}]}