{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84493,"databundleVersionId":9871156,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Table of Contents\n* [Load Data](#load)\n* [Weights](#weights)\n* [Features](#features)\n* [Targets](#targets)\n* [Distribution fits on target](#dist_fit)\n* [Correlation Features vs Features](#corr_features_features)\n* [Correlation Targets vs Features](#corr_target_features)","metadata":{}},{"cell_type":"code","source":"# install package for distribution fitting\n!pip install fitter","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-10-26T09:00:39.968469Z","iopub.execute_input":"2024-10-26T09:00:39.968907Z","iopub.status.idle":"2024-10-26T09:00:56.565368Z","shell.execute_reply.started":"2024-10-26T09:00:39.968862Z","shell.execute_reply":"2024-10-26T09:00:56.563921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport time\n\n# garbage collection\nimport gc\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# stats\nimport random\nimport scipy.stats\nfrom fitter import Fitter, get_common_distributions, get_distributions\n\n# other stuff\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-26T09:01:01.779896Z","iopub.execute_input":"2024-10-26T09:01:01.780374Z","iopub.status.idle":"2024-10-26T09:01:03.661375Z","shell.execute_reply.started":"2024-10-26T09:01:01.780325Z","shell.execute_reply":"2024-10-26T09:01:03.660223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show files\n!ls -l '../input/jane-street-real-time-market-data-forecasting'","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:01:08.307943Z","iopub.execute_input":"2024-10-26T09:01:08.308487Z","iopub.status.idle":"2024-10-26T09:01:09.509096Z","shell.execute_reply.started":"2024-10-26T09:01:08.308448Z","shell.execute_reply":"2024-10-26T09:01:09.507406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# details - training data\n!ls -lR '../input/jane-street-real-time-market-data-forecasting/train.parquet'","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:01:20.108748Z","iopub.execute_input":"2024-10-26T09:01:20.109189Z","iopub.status.idle":"2024-10-26T09:01:21.300565Z","shell.execute_reply.started":"2024-10-26T09:01:20.109144Z","shell.execute_reply":"2024-10-26T09:01:21.299116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\npd.set_option('display.max_rows', 100) # increase number of displayed rows\n\n# random seed\nmy_random_seed = 123\nrandom.seed(128)\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:01:30.836181Z","iopub.execute_input":"2024-10-26T09:01:30.836704Z","iopub.status.idle":"2024-10-26T09:01:30.844156Z","shell.execute_reply.started":"2024-10-26T09:01:30.836653Z","shell.execute_reply":"2024-10-26T09:01:30.842894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='load'></a>\n# Load Data","metadata":{}},{"cell_type":"code","source":"# load data\nt1 = time.time()\ndf_0 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=0/part-0.parquet')\ndf_1 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=1/part-0.parquet')\ndf_2 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=2/part-0.parquet')\ndf_3 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=3/part-0.parquet')\ndf_4 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=4/part-0.parquet')\ndf_5 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=5/part-0.parquet')\ndf_6 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=6/part-0.parquet')\ndf_7 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=7/part-0.parquet')\ndf_8 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=8/part-0.parquet')\ndf_9 = pl.read_parquet('../input/jane-street-real-time-market-data-forecasting/train.parquet/partition_id=9/part-0.parquet')\nt2 = time.time()\nprint('Elapsed time [s]:', np.round(t2-t1,4))","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:01:36.880749Z","iopub.execute_input":"2024-10-26T09:01:36.881194Z","iopub.status.idle":"2024-10-26T09:02:27.044166Z","shell.execute_reply.started":"2024-10-26T09:01:36.881151Z","shell.execute_reply":"2024-10-26T09:02:27.042835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# combine in one data frame\ndf = pl.concat([df_0,df_1,df_2,df_3,df_4,df_5,df_6,df_7,df_8,df_9])","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:02:29.854794Z","iopub.execute_input":"2024-10-26T09:02:29.855224Z","iopub.status.idle":"2024-10-26T09:02:29.881805Z","shell.execute_reply.started":"2024-10-26T09:02:29.855182Z","shell.execute_reply":"2024-10-26T09:02:29.880273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean up\ndel df_0,df_1,df_2,df_3,df_4,df_5,df_6,df_7,df_8,df_9\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:02:33.497958Z","iopub.execute_input":"2024-10-26T09:02:33.498388Z","iopub.status.idle":"2024-10-26T09:02:33.619189Z","shell.execute_reply.started":"2024-10-26T09:02:33.498347Z","shell.execute_reply":"2024-10-26T09:02:33.617879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preview\ndf.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:02:35.973820Z","iopub.execute_input":"2024-10-26T09:02:35.974254Z","iopub.status.idle":"2024-10-26T09:02:36.016607Z","shell.execute_reply.started":"2024-10-26T09:02:35.974213Z","shell.execute_reply":"2024-10-26T09:02:36.015463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data frame size\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:04:01.603940Z","iopub.execute_input":"2024-10-26T09:04:01.604560Z","iopub.status.idle":"2024-10-26T09:04:01.613511Z","shell.execute_reply.started":"2024-10-26T09:04:01.604482Z","shell.execute_reply":"2024-10-26T09:04:01.612001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='weights'></a>\n# Weights","metadata":{}},{"cell_type":"code","source":"# basic stats\ndf['weight'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:04:04.816634Z","iopub.execute_input":"2024-10-26T09:04:04.817415Z","iopub.status.idle":"2024-10-26T09:04:08.294730Z","shell.execute_reply.started":"2024-10-26T09:04:04.817367Z","shell.execute_reply":"2024-10-26T09:04:08.293570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot histogram\nplt.figure(figsize=(8,3))\nplt.hist(df['weight'], bins=100, color=default_color_1)\nplt.title('Distribution of weights')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:04:08.296960Z","iopub.execute_input":"2024-10-26T09:04:08.297332Z","iopub.status.idle":"2024-10-26T09:04:09.754501Z","shell.execute_reply.started":"2024-10-26T09:04:08.297293Z","shell.execute_reply":"2024-10-26T09:04:09.753325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# average weight by symbol\nweight_by_symbol = df.group_by('symbol_id').agg(pl.col('weight').mean())\nweight_by_symbol = weight_by_symbol.to_pandas().sort_values(by='symbol_id').reset_index(drop=True)\nweight_by_symbol","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:04:19.267997Z","iopub.execute_input":"2024-10-26T09:04:19.269230Z","iopub.status.idle":"2024-10-26T09:04:20.204832Z","shell.execute_reply.started":"2024-10-26T09:04:19.269180Z","shell.execute_reply":"2024-10-26T09:04:20.203497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='features'></a>\n# Features","metadata":{}},{"cell_type":"code","source":"features = ['feature_' + str(x).zfill(2) for x in range(78+1)]\nprint(features)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:05:52.202860Z","iopub.execute_input":"2024-10-26T09:05:52.203912Z","iopub.status.idle":"2024-10-26T09:05:52.210236Z","shell.execute_reply.started":"2024-10-26T09:05:52.203865Z","shell.execute_reply":"2024-10-26T09:05:52.208975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot feature distributions\nfor f in features:\n    plt.figure(figsize=(8,3))\n    plt.hist(df[f], bins=100, color=default_color_1)\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2024-10-26T09:09:23.801187Z","iopub.execute_input":"2024-10-26T09:09:23.801711Z","iopub.status.idle":"2024-10-26T09:13:17.537436Z","shell.execute_reply.started":"2024-10-26T09:09:23.801665Z","shell.execute_reply":"2024-10-26T09:13:17.536173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='targets'></a>\n# Targets","metadata":{}},{"cell_type":"code","source":"# target names\ntargets = ['responder_' + str(x) for x in range(8+1)]\nprint(targets)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:13:17.562070Z","iopub.execute_input":"2024-10-26T09:13:17.562444Z","iopub.status.idle":"2024-10-26T09:13:17.574642Z","shell.execute_reply.started":"2024-10-26T09:13:17.562405Z","shell.execute_reply":"2024-10-26T09:13:17.573211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats\ndf[targets].describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:23:37.444110Z","iopub.execute_input":"2024-10-26T09:23:37.445469Z","iopub.status.idle":"2024-10-26T09:24:04.251985Z","shell.execute_reply.started":"2024-10-26T09:23:37.445412Z","shell.execute_reply":"2024-10-26T09:24:04.250769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 Looks like an artificial cap at -/+5","metadata":{}},{"cell_type":"code","source":"# plot target distributions\nfor t in targets:\n    plt.figure(figsize=(8,3))\n    plt.hist(df[t], bins=100, color=default_color_3)\n    plt.title(t)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:24:06.459018Z","iopub.execute_input":"2024-10-26T09:24:06.460179Z","iopub.status.idle":"2024-10-26T09:24:19.343750Z","shell.execute_reply.started":"2024-10-26T09:24:06.460128Z","shell.execute_reply":"2024-10-26T09:24:19.342582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='dist_fit'></a>\n# Distribution fits on target","metadata":{}},{"cell_type":"code","source":"# this is our actual target\ntarget = 'responder_6'","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:24:22.449022Z","iopub.execute_input":"2024-10-26T09:24:22.449469Z","iopub.status.idle":"2024-10-26T09:24:22.454594Z","shell.execute_reply.started":"2024-10-26T09:24:22.449427Z","shell.execute_reply":"2024-10-26T09:24:22.453326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# available distributions in fitter packages\nprint(get_distributions())","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:24:25.252544Z","iopub.execute_input":"2024-10-26T09:24:25.253427Z","iopub.status.idle":"2024-10-26T09:24:25.277781Z","shell.execute_reply.started":"2024-10-26T09:24:25.253376Z","shell.execute_reply":"2024-10-26T09:24:25.276383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int(df.shape[0]*0.1)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:25:38.390036Z","iopub.execute_input":"2024-10-26T09:25:38.391000Z","iopub.status.idle":"2024-10-26T09:25:38.424386Z","shell.execute_reply.started":"2024-10-26T09:25:38.390892Z","shell.execute_reply":"2024-10-26T09:25:38.418584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try to fit a few distribution types to target 'responder_6'\n# we use a subset to achieve a reasonable run time\n\ndata = df[target].sample(1500000, seed=my_random_seed)\ndist_fitter = Fitter(data,\n                     distributions=[# Symmetric distributions 对成分布\n                         'norm', 't', 'laplace', 'logistic',\n                         # Heavy-tailed distributions 重尾分布 - 金融数据的典型特征，特别是在涉及收益率或价格变动时\n                         'cauchy', #适合建模极端事件\n                         'gennorm', 'hypsecant',\n                         # Flexible distributions 灵活分布\n                         'johnsonsu', 'skewnorm', 'pearson3',\n                         # Additional distributions for comparison\n                         'levy_stable', #适合建模金融市场中的极端波动\n                         'studentized_range', 'kappa4'], \n                     timeout=300)\ndist_fitter.fit()\nplt.figure(figsize=(12,5))\ndist_fitter.summary(13)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:41:34.844254Z","iopub.execute_input":"2024-10-26T09:41:34.844841Z","iopub.status.idle":"2024-10-26T09:47:47.921058Z","shell.execute_reply.started":"2024-10-26T09:41:34.844772Z","shell.execute_reply":"2024-10-26T09:47:47.919639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# just plotting the fitted PDFs\ndist_fitter.plot_pdf(Nbest=13, lw=1)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:47:59.646970Z","iopub.execute_input":"2024-10-26T09:47:59.648825Z","iopub.status.idle":"2024-10-26T09:48:00.190509Z","shell.execute_reply.started":"2024-10-26T09:47:59.648742Z","shell.execute_reply":"2024-10-26T09:48:00.189319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best fit\ndist_fitter.get_best()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:48:17.637268Z","iopub.execute_input":"2024-10-26T09:48:17.638326Z","iopub.status.idle":"2024-10-26T09:48:17.647631Z","shell.execute_reply.started":"2024-10-26T09:48:17.638261Z","shell.execute_reply":"2024-10-26T09:48:17.646201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['responder_6'].head(20)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T09:52:07.638140Z","iopub.execute_input":"2024-10-26T09:52:07.639664Z","iopub.status.idle":"2024-10-26T09:52:07.649045Z","shell.execute_reply.started":"2024-10-26T09:52:07.639600Z","shell.execute_reply":"2024-10-26T09:52:07.647616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1. 准备数据\nsample_x_y = df.sample(500000, seed=my_random_seed)\ny = sample_x_y['responder_6']\n\n# 2. 构建特征矩阵X - 只选择feature列\nfeature_cols = [col for col in sample_x_y.columns if col.startswith('feature_')]\nX = sample_x_y[feature_cols]\n\n# 3. 简单的缺失值处理 - 使用均值填充\nX = X.fill_null(strategy='mean')","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:28:33.773213Z","iopub.execute_input":"2024-10-26T11:28:33.773754Z","iopub.status.idle":"2024-10-26T11:28:40.090198Z","shell.execute_reply.started":"2024-10-26T11:28:33.773697Z","shell.execute_reply":"2024-10-26T11:28:40.088862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.select(pl.all().has_nulls())","metadata":{"execution":{"iopub.status.busy":"2024-10-26T10:43:42.584457Z","iopub.execute_input":"2024-10-26T10:43:42.584990Z","iopub.status.idle":"2024-10-26T10:43:42.601830Z","shell.execute_reply.started":"2024-10-26T10:43:42.584944Z","shell.execute_reply":"2024-10-26T10:43:42.600608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nfrom scipy import stats\nfrom sklearn.feature_selection import mutual_info_regression\n\nclass FeatureEvaluator:\n    def __init__(self, target_dist_type='johnsonsu'):\n        self.target_dist_type = target_dist_type\n    \n    def evaluate_features(self, X, y):\n        \"\"\"\n        综合评估特征，适配Polars DataFrame\n        \"\"\"\n        evaluations = {}\n        \n        # 转换为numpy数组以便计算\n        y_np = y.to_numpy()\n        \n        for column in X.columns:\n            # 获取特征并转换为numpy数组\n            feature_np = X[column].to_numpy()\n            \n            # 1. 基础统计量\n            basic_stats = self._calculate_basic_stats(feature_np)\n            \n            # 2. 与目标变量的关系评估\n            relationship_stats = self._evaluate_relationship(feature_np, y_np)\n            \n            # 3. 分布特征相似度\n            distribution_stats = self._evaluate_distribution_similarity(feature_np, y_np)\n            \n            # 综合评分\n            composite_score = self._calculate_composite_score(\n                basic_stats, relationship_stats, distribution_stats)\n            \n            evaluations[column] = {\n                'basic_stats': basic_stats,\n                'relationship_stats': relationship_stats,\n                'distribution_stats': distribution_stats,\n                'composite_score': composite_score\n            }\n        \n        return self._rank_features(evaluations)\n    \n    def _calculate_basic_stats(self, feature):\n        \"\"\"\n        计算基础统计量\n        \"\"\"\n        return {\n            'mean': np.mean(feature),\n            'std': np.std(feature),\n            'skew': stats.skew(feature),\n            'kurtosis': stats.kurtosis(feature)\n        }\n    \n    def _evaluate_relationship(self, feature, y):\n        \"\"\"\n        评估特征与目标变量的关系\n        \"\"\"\n        # 计算多种相关性指标\n        pearson = stats.pearsonr(feature, y)[0]\n        spearman = stats.spearmanr(feature, y)[0]\n        mutual_info = mutual_info_regression(\n            feature.reshape(-1, 1), y, random_state=42)[0]\n        \n        # 计算尾部相关性\n        tail_corr = self._calculate_tail_correlation(feature, y)\n        \n        return {\n            'pearson': pearson,\n            'spearman': spearman,\n            'mutual_info': mutual_info,\n            'tail_correlation': tail_corr\n        }\n    \n    def _calculate_tail_correlation(self, feature, y, quantile=0.1):\n        \"\"\"\n        计算尾部相关性\n        \"\"\"\n        # 获取尾部样本\n        feature_lower = np.quantile(feature, quantile)\n        feature_upper = np.quantile(feature, 1-quantile)\n        y_lower = np.quantile(y, quantile)\n        y_upper = np.quantile(y, 1-quantile)\n        \n        lower_mask = (feature <= feature_lower) & (y <= y_lower)\n        upper_mask = (feature >= feature_upper) & (y >= y_upper)\n        \n        # 计算尾部相关性得分\n        tail_score = (np.sum(lower_mask) + np.sum(upper_mask)) / \\\n                    (2 * len(feature) * quantile)\n                    \n        return tail_score\n    \n    def _evaluate_distribution_similarity(self, feature, y):\n        \"\"\"\n        评估分布相似度\n        \"\"\"\n        # 标准化以便比较\n        feature_normalized = (feature - np.mean(feature)) / np.std(feature)\n        y_normalized = (y - np.mean(y)) / np.std(y)\n        \n        # 比较分布特征\n        similarity = {\n            'skew_similarity': 1 / (1 + abs(stats.skew(feature_normalized) - \n                                          stats.skew(y_normalized))),\n            'kurtosis_similarity': 1 / (1 + abs(stats.kurtosis(feature_normalized) - \n                                              stats.kurtosis(y_normalized))),\n        }\n        \n        return similarity\n    \n    def _calculate_composite_score(self, basic_stats, relationship_stats, \n                                 distribution_stats):\n        \"\"\"\n        计算综合评分\n        \"\"\"\n        # 1. 关系得分 (40%)\n        relationship_score = 0.4 * (\n            0.3 * abs(relationship_stats['pearson']) +\n            0.3 * abs(relationship_stats['spearman']) +\n            0.2 * relationship_stats['mutual_info'] +\n            0.2 * relationship_stats['tail_correlation']\n        )\n        \n        # 2. 分布相似度得分 (30%)\n        distribution_score = 0.3 * (\n            0.5 * distribution_stats['skew_similarity'] +\n            0.5 * distribution_stats['kurtosis_similarity']\n        )\n        \n        # 3. 基础统计得分 (30%)\n        basic_score = 0.3 * (\n            1 / (1 + abs(basic_stats['skew'])) * 0.5 +\n            1 / (1 + abs(basic_stats['kurtosis'] - 3)) * 0.5\n        )\n        \n        return relationship_score + distribution_score + basic_score\n    \n    def _rank_features(self, evaluations):\n        \"\"\"\n        对特征进行排名并生成报告\n        \"\"\"\n        # 按综合得分排序\n        ranked_features = sorted(\n            evaluations.items(),\n            key=lambda x: x[1]['composite_score'],\n            reverse=True\n        )\n        \n        # 生成报告\n        report = {\n            'rankings': ranked_features,\n            'top_features': [f[0] for f in ranked_features[:10]],\n            'feature_groups': self._group_features(evaluations)\n        }\n        \n        return report\n    \n    def _group_features(self, evaluations):\n        \"\"\"\n        将特征分组\n        \"\"\"\n        groups = {\n            'strong_predictors': [],\n            'tail_predictors': [],\n            'noise_features': [],\n            'potential_transforms': []\n        }\n        \n        for feature, eval_data in evaluations.items():\n            # 强预测特征\n            if eval_data['composite_score'] > 0.7:\n                groups['strong_predictors'].append(feature)\n            \n            # 尾部预测特征\n            elif eval_data['relationship_stats']['tail_correlation'] > 0.3:\n                groups['tail_predictors'].append(feature)\n            \n            # 需要转换的特征\n            elif abs(eval_data['basic_stats']['skew']) > 1:\n                groups['potential_transforms'].append(feature)\n            \n            # 噪声特征\n            elif eval_data['composite_score'] < 0.3:\n                groups['noise_features'].append(feature)\n        \n        return groups\n\n# 使用评估器\ndef print_evaluation_results(evaluation_results):\n    \"\"\"\n    打印评估结果\n    \"\"\"\n    print(\"\\nTop 10 Features:\")\n    print(\"-\" * 50)\n    for feature in evaluation_results['top_features']:\n        score = next(e[1]['composite_score'] \n                    for e in evaluation_results['rankings'] \n                    if e[0] == feature)\n        print(f\"{feature}: {score:.4f}\")\n\n    print(\"\\nFeature Groups:\")\n    print(\"-\" * 50)\n    for group, features in evaluation_results['feature_groups'].items():\n        print(f\"\\n{group.replace('_', ' ').title()}:\")\n        print(f\"Count: {len(features)}\")\n        if len(features) > 0:\n            print(f\"Examples: {', '.join(features[:3])}\")\n\n# 运行评估\nevaluator = FeatureEvaluator(target_dist_type='johnsonsu')\nevaluation_results = evaluator.evaluate_features(X, y)\nprint_evaluation_results(evaluation_results)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:28:41.436616Z","iopub.execute_input":"2024-10-26T11:28:41.437086Z","iopub.status.idle":"2024-10-26T11:43:20.306674Z","shell.execute_reply.started":"2024-10-26T11:28:41.437046Z","shell.execute_reply":"2024-10-26T11:43:20.305304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Whole Report of the evaluation results","metadata":{}},{"cell_type":"code","source":"evaluation_results","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:50:16.308826Z","iopub.execute_input":"2024-10-26T11:50:16.309349Z","iopub.status.idle":"2024-10-26T11:50:16.373766Z","shell.execute_reply.started":"2024-10-26T11:50:16.309290Z","shell.execute_reply":"2024-10-26T11:50:16.372578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the Top 10 features, we shall use the corrlation to find whether other is significantly correlated to those for feature engineering later.","metadata":{}},{"cell_type":"markdown","source":"# Correlation Features vs Features","metadata":{}},{"cell_type":"code","source":"# init correlation matrix\ncorr_matrix = np.ones((79,79), dtype='float64')\n\n# calc correlations pairwise\nj = 0 # target index\nfor t in features:\n    print('Calculating for target', t)\n    i = 0 # feature index\n    for f in features:\n        corr_ft = df.select(pl.corr(f, t, method='pearson'))[0,0]\n        corr_matrix[i][j] = corr_ft\n        i = i + 1\n    j = j + 1","metadata":{"execution":{"iopub.status.busy":"2024-10-26T12:08:45.800272Z","iopub.execute_input":"2024-10-26T12:08:45.800780Z","iopub.status.idle":"2024-10-26T13:02:49.829177Z","shell.execute_reply.started":"2024-10-26T12:08:45.800736Z","shell.execute_reply":"2024-10-26T13:02:49.824787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert to data frame\ncorr_features_features = pd.DataFrame(corr_matrix, columns=features)\ncorr_features_features.index = features","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:05:50.624946Z","iopub.execute_input":"2024-10-26T13:05:50.625911Z","iopub.status.idle":"2024-10-26T13:05:50.633107Z","shell.execute_reply.started":"2024-10-26T13:05:50.625857Z","shell.execute_reply":"2024-10-26T13:05:50.631644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show correlations between features and targets\ncorr_features_features","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:05:55.197143Z","iopub.execute_input":"2024-10-26T13:05:55.197648Z","iopub.status.idle":"2024-10-26T13:05:55.520838Z","shell.execute_reply.started":"2024-10-26T13:05:55.197600Z","shell.execute_reply":"2024-10-26T13:05:55.519557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize correlations\nplt.figure(figsize=(12,12))\nsns.heatmap(corr_features_features, cmap='RdYlGn',\n            linewidths=0.5, linecolor='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:06:39.761843Z","iopub.execute_input":"2024-10-26T13:06:39.762697Z","iopub.status.idle":"2024-10-26T13:06:41.143000Z","shell.execute_reply.started":"2024-10-26T13:06:39.762648Z","shell.execute_reply":"2024-10-26T13:06:41.141750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\n\n# Define features to highlight\nhighlighted_features = [\n    'feature_08', 'feature_44', 'feature_36', 'feature_04', 'feature_20',\n    'feature_41', 'feature_43', 'feature_33', 'feature_61', 'feature_40'\n]\n\n# Set up the plot\nplt.figure(figsize=(12, 12))\n\n# Create a mask that highlights specific rows\nmask = np.zeros_like(corr_features_features, dtype=bool)\nmask[~corr_features_features.index.isin(highlighted_features), :] = True\n\n# Plot the heatmap, setting mask and custom color for highlighted rows\nsns.heatmap(corr_features_features, cmap='RdYlGn', linewidths=0.5, linecolor='black',\n            mask=mask, cbar_kws={\"shrink\": .8}, fmt=\".2f\")\n\n# Overlay another heatmap to make highlighted rows stand out with a strong color\nsns.heatmap(corr_features_features, mask=~mask, cmap=\"coolwarm\", cbar=False,\n            linewidths=0.8, linecolor='black', alpha=0.75)\n\n# Display plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:08:44.966876Z","iopub.execute_input":"2024-10-26T13:08:44.967421Z","iopub.status.idle":"2024-10-26T13:08:46.280300Z","shell.execute_reply.started":"2024-10-26T13:08:44.967374Z","shell.execute_reply":"2024-10-26T13:08:46.278844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='corr_target_features'></a>\n# Correlation Targets vs Features","metadata":{}},{"cell_type":"code","source":"# init correlation matrix\ncorr_matrix = np.ones((79,9), dtype='float64')\n\n# calc correlations pairwise\nj = 0 # target index\nfor t in targets:\n    print('Calculating for target', t)\n    i = 0 # feature index\n    for f in features:\n        corr_ft = df.select(pl.corr(f, t, method='pearson'))[0,0]\n        corr_matrix[i][j] = corr_ft\n        i = i + 1\n    j = j + 1","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:45:05.567987Z","iopub.execute_input":"2024-10-26T11:45:05.568549Z","iopub.status.idle":"2024-10-26T11:50:16.220335Z","shell.execute_reply.started":"2024-10-26T11:45:05.568486Z","shell.execute_reply":"2024-10-26T11:50:16.218952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert to data frame\ncorr_targets_features = pd.DataFrame(corr_matrix, columns=targets)\ncorr_targets_features.index = features","metadata":{"execution":{"iopub.status.busy":"2024-10-26T11:50:16.222712Z","iopub.execute_input":"2024-10-26T11:50:16.223149Z","iopub.status.idle":"2024-10-26T11:50:16.236412Z","shell.execute_reply.started":"2024-10-26T11:50:16.223106Z","shell.execute_reply":"2024-10-26T11:50:16.234641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show correlations between features and targets\ncorr_targets_features","metadata":{"execution":{"iopub.status.busy":"2024-10-26T12:00:53.632038Z","iopub.execute_input":"2024-10-26T12:00:53.632618Z","iopub.status.idle":"2024-10-26T12:00:53.695868Z","shell.execute_reply.started":"2024-10-26T12:00:53.632569Z","shell.execute_reply":"2024-10-26T12:00:53.694435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize correlations\nplt.figure(figsize=(6,16))\nsns.heatmap(corr_targets_features, cmap='RdYlGn',\n            linewidths=0.5, linecolor='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T12:00:58.446610Z","iopub.execute_input":"2024-10-26T12:00:58.447561Z","iopub.status.idle":"2024-10-26T12:00:59.531178Z","shell.execute_reply.started":"2024-10-26T12:00:58.447490Z","shell.execute_reply":"2024-10-26T12:00:59.529905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export result\ncorr_targets_features.to_csv('corr_targets_features.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:11:35.546498Z","iopub.execute_input":"2024-10-26T13:11:35.547039Z","iopub.status.idle":"2024-10-26T13:11:35.564724Z","shell.execute_reply.started":"2024-10-26T13:11:35.546996Z","shell.execute_reply":"2024-10-26T13:11:35.563632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize an example pair\nf = 'feature_08'\nt = 'responder_6'\nsns.jointplot(data=df, x=f, y=t,\n              kind='hist')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-26T13:12:25.496167Z","iopub.execute_input":"2024-10-26T13:12:25.496703Z","iopub.status.idle":"2024-10-26T13:15:19.975337Z","shell.execute_reply.started":"2024-10-26T13:12:25.496654Z","shell.execute_reply":"2024-10-26T13:15:19.974225Z"},"trusted":true},"execution_count":null,"outputs":[]}]}