{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":12044552,"sourceType":"datasetVersion","datasetId":7579435},{"sourceId":12056012,"sourceType":"datasetVersion","datasetId":7587701}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport joblib\nimport gc\nimport os\nimport json\nfrom typing import Dict, List, Tuple, Optional, Any, Set\nfrom dataclasses import dataclass\nfrom abc import ABC, abstractmethod\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.linear_model import Ridge\nfrom sklearn.base import clone\nfrom scipy.stats import pearsonr, spearmanr, kendalltau\nfrom xgboost import XGBRegressor\nfrom koolbox import Trainer\nimport optuna\nfrom dataclasses import field\nimport time\nfrom pathlib import Path\nimport itertools\nfrom tqdm import tqdm\n\nwarnings.filterwarnings(\"ignore\")\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n\nprint(\"Libraries imported successfully!\")\n\n@dataclass\nclass Config:\n    \"\"\"Configuration class for high-frequency trading factor selection.\"\"\"\n    \n    # Data paths\n    X_train_path: str = \"../kaggle/input/X_train.parquet\"\n    y_train_path: str = \"../kaggle/input/y_train.parquet\"\n    X_test_path: str = \"../kaggle/input/X_test.parquet\"\n    sample_sub_path: str = \"../kaggle/input/sample_submission.csv\"\n    \n    # 单因子分析文件路径\n    single_factor_file: str = \"D:/jacky/kaggle/drw/results/xgb/filtered_factors_IC_IR.csv\"\n\n    # Output paths\n    models_dir: str = \"/kaggle/working/models/\"\n    results_dir: str = \"/kaggle/working/results/\"\n    top_results_dir: str = \"/kaggle/working/results/xgb/\"\n    top_results_file: str = \"/kaggle/working/results/xgb/top1000.csv\"\n    factor_monitoring_file: str = \"/kaggle/working/results/xgb/factor_monitoring.csv\"\n    \n    # Model parameters\n    target: str = \"label\"\n    n_folds: int = 5\n    seed: int = 42\n    \n    # HFT-optimized search parameters (根据高频交易文档调整)\n    max_factors: int = 15  # 高频交易推荐最大15个因子\n    min_factors: int = 5   # 高频交易推荐最小5个因子\n    optimal_basic: int = 8   # 基础模型：5-8个核心因子\n    optimal_enhanced: int = 12  # 增强模型：8-12个精选因子\n    \n    max_iterations: int = 150\n    early_stopping_rounds: int = 12\n    top_k_results: int = 1000\n    \n    # 高频交易专用参数\n    factor_correlation_threshold: float = 0.4  # 因子相关性阈值\n    min_pearson_threshold: float = 0.03  # 最小Pearson相关性\n    min_ic_threshold: float = 0.02  # 最小IC阈值\n    rolling_window_size: int = 30  # 滚动窗口大小（分钟）\n    \n    # 单因子筛选参数\n    top_single_factors: int = 60  # 增加候选因子池\n    score_threshold: float = 0.2\n    \n    # GPU settings\n    use_gpu: bool = True\n    gpu_id: int = 0\n    \n    # 内存优化参数\n    batch_size: int = 50  # 分批处理因子\n    sample_size_for_search: int = 100000  # 用于搜索的样本数量\n    \n    def __post_init__(self):\n        os.makedirs(self.models_dir, exist_ok=True)\n        os.makedirs(self.results_dir, exist_ok=True)\n        os.makedirs(self.top_results_dir, exist_ok=True)\n\nclass HFTFactorAnalyzer:\n    \"\"\"高频交易专用因子分析器\"\"\"\n    \n    def __init__(self, config: Config):\n        self.config = config\n        self.factor_scores = self._load_single_factor_results()\n        \n    def _load_single_factor_results(self) -> pd.DataFrame:\n        \"\"\"加载单因子分析结果\"\"\"\n        try:\n            df = pd.read_csv(self.config.single_factor_file)\n            print(f\"Loaded single factor analysis with {len(df)} factors\")\n            \n            # 按高频交易标准过滤因子\n            df = df[\n                (df['Pearson'].abs() >= self.config.min_pearson_threshold) |\n                (df['IC'].abs() >= self.config.min_ic_threshold)\n            ]\n            \n            print(f\"After HFT filtering: {len(df)} factors remain\")\n            print(f\"Top 5 factors by Total_Score:\")\n            print(df.head()[['Factors', 'Total_Score', 'IC', 'IR', 'ICIR']])\n            \n            return df\n        except Exception as e:\n            print(f\"Error loading single factor file: {e}\")\n            return pd.DataFrame()\n    \n    def get_hft_factor_tiers(self) -> Dict[str, List[str]]:\n        \"\"\"按高频交易标准分层因子\"\"\"\n        if len(self.factor_scores) == 0:\n            return {\"tier1\": [f'X{i}' for i in range(20)]}\n        \n        sorted_factors = self.factor_scores.sort_values('Total_Score', ascending=False)\n        n_factors = len(sorted_factors)\n        \n        # 高频交易分层策略\n        tier1_size = min(15, max(8, n_factors // 4))  # 核心因子 8-15个\n        tier2_size = min(30, max(15, n_factors // 2))  # 候选因子 15-30个\n        \n        return {\n            \"tier1\": sorted_factors.head(tier1_size)['Factors'].tolist(),\n            \"tier2\": sorted_factors.iloc[tier1_size:tier2_size]['Factors'].tolist(),\n            \"tier3\": sorted_factors.iloc[tier2_size:]['Factors'].tolist()\n        }\n    \n    def calculate_factor_correlation_matrix(self, X: pd.DataFrame, factors: List[str]) -> pd.DataFrame:\n        \"\"\"计算因子相关性矩阵\"\"\"\n        available_factors = [f for f in factors if f in X.columns]\n        if len(available_factors) == 0:\n            return pd.DataFrame()\n        \n        return X[available_factors].corr().abs()\n\nclass HFTFactorSearchResults:\n    \"\"\"高频交易因子搜索结果管理器\"\"\"\n    \n    def __init__(self, config: Config):\n        self.config = config\n        self.results_file = config.top_results_file\n        self.monitoring_file = config.factor_monitoring_file\n        self.max_results = config.top_k_results\n        self.results_df = self._load_existing_results()\n    \n    def _load_existing_results(self) -> pd.DataFrame:\n        \"\"\"加载已存在的结果文件\"\"\"\n        if os.path.exists(self.results_file):\n            try:\n                df = pd.read_csv(self.results_file)\n                print(f\"Loaded {len(df)} existing results from {self.results_file}\")\n                return df\n            except Exception as e:\n                print(f\"Error loading results file: {e}\")\n        \n        return pd.DataFrame(columns=[\n            'factors', 'score', 'std', 'num_factors', 'timestamp',\n            'factor_category', 'avg_correlation', 'ic_stability'\n        ])\n    \n    def add_result(self, factors: List[str], score: float, std: float, \n                   factor_category: str = \"unknown\", avg_correlation: float = 0.0,\n                   ic_stability: float = 0.0):\n        \"\"\"添加新结果（增强版）\"\"\"\n        new_row = {\n            'factors': ','.join(sorted(factors)),\n            'score': score,\n            'std': std,\n            'num_factors': len(factors),\n            'timestamp': time.time(),\n            'factor_category': factor_category,\n            'avg_correlation': avg_correlation,\n            'ic_stability': ic_stability\n        }\n        \n        self.results_df = pd.concat([self.results_df, pd.DataFrame([new_row])], ignore_index=True)\n        \n        # 按HFT标准排序：优先考虑稳定性和低相关性\n        self.results_df['hft_score'] = (\n            self.results_df['score'] * 0.6 +  # 预测能力\n            (1 - self.results_df['avg_correlation']) * 0.2 +  # 低相关性\n            self.results_df['ic_stability'] * 0.2  # IC稳定性\n        )\n        \n        self.results_df = self.results_df.sort_values('hft_score', ascending=False)\n        \n        if len(self.results_df) > self.max_results:\n            self.results_df = self.results_df.head(self.max_results)\n        \n        self._save_results()\n        \n        print(f\"Added result: score={score:.6f}, factors={len(factors)}, \"\n              f\"hft_score={self.results_df.iloc[0]['hft_score']:.6f}\")\n    \n    def _save_results(self):\n        \"\"\"保存结果到文件\"\"\"\n        try:\n            self.results_df.to_csv(self.results_file, index=False)\n        except Exception as e:\n            print(f\"Error saving results: {e}\")\n    \n    def get_hft_summary(self):\n        \"\"\"获取高频交易专用结果摘要\"\"\"\n        if len(self.results_df) == 0:\n            return \"No results yet.\"\n        \n        # 按因子数量分组统计\n        category_stats = {}\n        for num_factors in range(self.config.min_factors, self.config.max_factors + 1):\n            subset = self.results_df[self.results_df['num_factors'] == num_factors]\n            if len(subset) > 0:\n                category_stats[f\"{num_factors}_factors\"] = {\n                    'count': len(subset),\n                    'best_score': subset['score'].max(),\n                    'best_hft_score': subset['hft_score'].max(),\n                    'avg_correlation': subset['avg_correlation'].mean()\n                }\n        \n        best_basic = self.results_df[\n            self.results_df['num_factors'] <= self.config.optimal_basic\n        ].iloc[0] if len(self.results_df) > 0 else None\n        \n        best_enhanced = self.results_df[\n            (self.results_df['num_factors'] > self.config.optimal_basic) &\n            (self.results_df['num_factors'] <= self.config.optimal_enhanced)\n        ].iloc[0] if len(self.results_df) > 0 else None\n        \n        summary = f\"\"\"\n        ========== HFT FACTOR SELECTION SUMMARY ==========\n        Total combinations tested: {len(self.results_df)}\n        \n        🏆 BEST BASIC MODEL (≤{self.config.optimal_basic} factors):\n        {f\"Score: {best_basic['score']:.6f}, HFT Score: {best_basic['hft_score']:.6f}\" if best_basic is not None else \"No results\"}\n        {f\"Factors ({best_basic['num_factors']}): {best_basic['factors']}\" if best_basic is not None else \"\"}\n        \n        🚀 BEST ENHANCED MODEL ({self.config.optimal_basic+1}-{self.config.optimal_enhanced} factors):\n        {f\"Score: {best_enhanced['score']:.6f}, HFT Score: {best_enhanced['hft_score']:.6f}\" if best_enhanced is not None else \"No results\"}\n        {f\"Factors ({best_enhanced['num_factors']}): {best_enhanced['factors'][:100]}...\" if best_enhanced is not None else \"\"}\n        \n        📊 Results by factor count:\n        \"\"\"\n        \n        for category, stats in category_stats.items():\n            summary += f\"\\n{category}: {stats['count']} combinations, best_score: {stats['best_score']:.6f}\"\n        \n        summary += \"\\n\" + \"=\" * 50\n        \n        return summary\n\nclass HFTGreedyFactorSelector:\n    \"\"\"高频交易专用贪心因子选择器\"\"\"\n    \n    def __init__(self, config: Config):\n        self.config = config\n        self.results_manager = HFTFactorSearchResults(config)\n        self.factor_analyzer = HFTFactorAnalyzer(config)\n        self.factor_tiers = self.factor_analyzer.get_hft_factor_tiers()\n        self.candidate_factors = (\n            self.factor_tiers.get(\"tier1\", []) + \n            self.factor_tiers.get(\"tier2\", [])\n        )[:config.top_single_factors]\n        \n        print(f\"HFT Factor Selector initialized:\")\n        print(f\"- Tier1 factors: {len(self.factor_tiers.get('tier1', []))}\")\n        print(f\"- Tier2 factors: {len(self.factor_tiers.get('tier2', []))}\")\n        print(f\"- Total candidates: {len(self.candidate_factors)}\")\n        \n    def _evaluate_factors_hft(self, X: pd.DataFrame, y: pd.Series, factors: List[str]) -> Dict[str, float]:\n        \"\"\"高频交易专用因子评估\"\"\"\n        if len(factors) == 0:\n            return {'score': -np.inf, 'std': np.inf, 'avg_correlation': 1.0, 'ic_stability': 0.0}\n        \n        # 检查因子可用性\n        available_factors = [f for f in factors if f in X.columns]\n        if len(available_factors) == 0:\n            return {'score': -np.inf, 'std': np.inf, 'avg_correlation': 1.0, 'ic_stability': 0.0}\n        \n        factors = available_factors\n        X_subset = X[factors]\n        \n        # 计算因子间相关性\n        if len(factors) > 1:\n            corr_matrix = X_subset.corr().abs()\n            # 排除对角线，计算平均相关性\n            mask = ~np.eye(corr_matrix.shape[0], dtype=bool)\n            avg_correlation = corr_matrix.values[mask].mean()\n        else:\n            avg_correlation = 0.0\n        \n        # 轻量级XGBoost配置（适合高频交易）\n        model_params = {\n            \"tree_method\": \"gpu_hist\" if self.config.use_gpu else \"hist\",\n            \"gpu_id\": self.config.gpu_id if self.config.use_gpu else None,\n            \"random_state\": self.config.seed,\n            \"n_jobs\": -1,\n            \"verbosity\": 0,\n            \"n_estimators\": 100,  # 较少的树，提高速度\n            \"max_depth\": 4,       # 浅层树\n            \"learning_rate\": 0.1,\n            \"subsample\": 0.8,\n            \"colsample_bytree\": 0.8,\n            \"reg_alpha\": 1,\n            \"reg_lambda\": 1,\n        }\n        \n        try:\n            # 时间序列交叉验证（更适合高频数据）\n            trainer = Trainer(\n                XGBRegressor(**model_params),\n                cv=TimeSeriesSplit(n_splits=self.config.n_folds),\n                metric=lambda y_true, y_pred: pearsonr(y_true, y_pred)[0],\n                task=\"regression\",\n                verbose=False\n            )\n            \n            trainer.fit(X_subset, y)\n            fold_scores = trainer.fold_scores\n            mean_score = np.mean(fold_scores)\n            std_score = np.std(fold_scores)\n            \n            # 计算IC稳定性（滚动IC的标准差的倒数）\n            ic_stability = 1.0 / (std_score + 1e-6) if std_score > 0 else 0.0\n            \n            del trainer\n            gc.collect()\n            \n            return {\n                'score': mean_score,\n                'std': std_score,\n                'avg_correlation': avg_correlation,\n                'ic_stability': ic_stability\n            }\n            \n        except Exception as e:\n            print(f\"Error evaluating factors {factors}: {e}\")\n            return {'score': -np.inf, 'std': np.inf, 'avg_correlation': 1.0, 'ic_stability': 0.0}\n    \n    def smart_exhaustive_search(self, X: pd.DataFrame, y: pd.Series, k: int):\n        \"\"\"智能穷举搜索（加入相关性过滤）\"\"\"\n        print(f\"Smart exhaustive search for {k}-factor combinations...\")\n        \n        # 使用tier1因子进行穷举\n        top_factors = self.factor_tiers.get(\"tier1\", self.candidate_factors[:20])\n        \n        if len(top_factors) <= 20 or k <= 4:\n            # 小规模穷举\n            combinations = list(itertools.combinations(top_factors, k))\n            print(f\"Testing all {len(combinations)} combinations of {k} factors\")\n        else:\n            # 智能采样：优先选择相关性低的组合\n            print(f\"Smart sampling from {len(top_factors)} factors...\")\n            \n            # 计算因子相关性矩阵\n            corr_matrix = self.factor_analyzer.calculate_factor_correlation_matrix(X, top_factors)\n            \n            # 贪心选择低相关性组合\n            combinations = []\n            max_combinations = 500\n            \n            for _ in range(max_combinations):\n                # 随机选择第一个因子\n                selected = [np.random.choice(top_factors)]\n                \n                # 贪心添加相关性最低的因子\n                remaining = [f for f in top_factors if f not in selected]\n                \n                while len(selected) < k and remaining:\n                    # 计算每个候选因子与已选因子的平均相关性\n                    avg_corrs = []\n                    for candidate in remaining:\n                        if candidate in corr_matrix.index and all(s in corr_matrix.columns for s in selected):\n                            avg_corr = corr_matrix.loc[candidate, selected].mean()\n                        else:\n                            avg_corr = 0.5  # 默认值\n                        avg_corrs.append(avg_corr)\n                    \n                    # 选择相关性最低的因子\n                    min_corr_idx = np.argmin(avg_corrs)\n                    selected.append(remaining[min_corr_idx])\n                    remaining.pop(min_corr_idx)\n                \n                if len(selected) == k:\n                    combinations.append(tuple(selected))\n            \n            # 去重\n            combinations = list(set(combinations))\n            print(f\"Generated {len(combinations)} diverse combinations\")\n        \n        # 评估组合\n        best_combinations = []\n        \n        for i, factors in enumerate(combinations):\n            if i % 50 == 0:\n                print(f\"Progress: {i}/{len(combinations)}\")\n            \n            factors_list = list(factors)\n            \n            if self.results_manager.results_df is not None:\n                factors_str = ','.join(sorted(factors_list))\n                if factors_str in self.results_manager.results_df['factors'].values:\n                    continue\n            \n            metrics = self._evaluate_factors_hft(X, y, factors_list)\n            \n            # 高频交易专用过滤\n            if (metrics['score'] > 0.01 and \n                metrics['avg_correlation'] < self.config.factor_correlation_threshold):\n                \n                factor_category = f\"exhaustive_{k}f\"\n                self.results_manager.add_result(\n                    factors_list, \n                    metrics['score'], \n                    metrics['std'],\n                    factor_category,\n                    metrics['avg_correlation'],\n                    metrics['ic_stability']\n                )\n                \n                best_combinations.append((factors_list, metrics['score']))\n        \n        if best_combinations:\n            best_combinations.sort(key=lambda x: x[1], reverse=True)\n            return best_combinations[:20]\n        \n        return []\n    \n    def hft_forward_selection(self, X: pd.DataFrame, y: pd.Series, \n                             start_factors: Optional[List[str]] = None) -> Tuple[List[str], float]:\n        \"\"\"高频交易专用前向选择\"\"\"\n        print(\"Starting HFT-optimized forward selection...\")\n        \n        if start_factors is None:\n            # 从最佳小组合开始\n            selected_factors = self.factor_tiers.get(\"tier1\", self.candidate_factors)[:self.config.min_factors]\n        else:\n            selected_factors = start_factors.copy()\n        \n        remaining_factors = [f for f in self.candidate_factors if f not in selected_factors]\n        \n        current_metrics = self._evaluate_factors_hft(X, y, selected_factors)\n        current_score = current_metrics['score']\n        print(f\"Starting score: {current_score:.6f}, correlation: {current_metrics['avg_correlation']:.3f}\")\n        \n        no_improvement_count = 0\n        \n        while (len(selected_factors) < self.config.max_factors and \n               no_improvement_count < self.config.early_stopping_rounds):\n            \n            print(f\"\\n--- HFT Forward Selection: {len(selected_factors)} -> {len(selected_factors)+1} factors ---\")\n            \n            best_candidate = None\n            best_candidate_score = current_score\n            best_candidate_metrics = None\n            \n            # 计算当前因子的相关性矩阵\n            if len(selected_factors) > 0:\n                current_corr_matrix = X[selected_factors].corr().abs()\n            else:\n                current_corr_matrix = None\n            \n            # 优先测试低相关性的候选因子\n            candidate_scores = []\n            for factor in remaining_factors[:30]:  # 限制候选数量以提高效率\n                if factor in X.columns:\n                    # 计算与已选因子的相关性\n                    if current_corr_matrix is not None and factor in X.columns:\n                        factor_corrs = []\n                        for selected in selected_factors:\n                            if selected in X.columns:\n                                corr = X[factor].corr(X[selected])\n                                factor_corrs.append(abs(corr) if not np.isnan(corr) else 0)\n                        avg_corr = np.mean(factor_corrs) if factor_corrs else 0\n                    else:\n                        avg_corr = 0\n                    \n                    # 综合评分：优先选择相关性低的因子\n                    candidate_scores.append((factor, avg_corr))\n            \n            # 按相关性排序，优先测试低相关性因子\n            candidate_scores.sort(key=lambda x: x[1])\n            \n            for i, (factor, avg_corr) in enumerate(candidate_scores):\n                if i % 10 == 0:\n                    print(f\"Testing factor {i}/{len(candidate_scores)}: {factor} (avg_corr: {avg_corr:.3f})\")\n                \n                # 跳过高相关性因子\n                if avg_corr > self.config.factor_correlation_threshold:\n                    continue\n                \n                candidate_factors = selected_factors + [factor]\n                metrics = self._evaluate_factors_hft(X, y, candidate_factors)\n                \n                # 保存结果\n                factor_category = f\"forward_{len(candidate_factors)}f\"\n                self.results_manager.add_result(\n                    candidate_factors,\n                    metrics['score'],\n                    metrics['std'],\n                    factor_category,\n                    metrics['avg_correlation'],\n                    metrics['ic_stability']\n                )\n                \n                # 多目标优化：分数提升 + 相关性控制\n                score_improvement = metrics['score'] - current_score\n                correlation_penalty = max(0, metrics['avg_correlation'] - self.config.factor_correlation_threshold)\n                \n                adjusted_score = metrics['score'] - correlation_penalty * 0.1\n                \n                if adjusted_score > best_candidate_score:\n                    best_candidate = factor\n                    best_candidate_score = adjusted_score\n                    best_candidate_metrics = metrics\n                    print(f\"New best candidate: {factor}, score: {metrics['score']:.6f}, corr: {metrics['avg_correlation']:.3f}\")\n            \n            # 添加最佳候选因子\n            if best_candidate is not None and best_candidate_metrics['score'] > current_score:\n                selected_factors.append(best_candidate)\n                remaining_factors.remove(best_candidate)\n                current_score = best_candidate_metrics['score']\n                current_metrics = best_candidate_metrics\n                no_improvement_count = 0\n                \n                print(f\"✅ Added factor: {best_candidate}\")\n                print(f\"   New score: {current_score:.6f}\")\n                print(f\"   Avg correlation: {current_metrics['avg_correlation']:.3f}\")\n                print(f\"   IC stability: {current_metrics['ic_stability']:.3f}\")\n            else:\n                no_improvement_count += 1\n                print(f\"❌ No improvement ({no_improvement_count}/{self.config.early_stopping_rounds})\")\n        \n        return selected_factors, current_score\n    \n    def run_hft_comprehensive_search(self, X: pd.DataFrame, y: pd.Series):\n        \"\"\"运行高频交易全面因子搜索\"\"\"\n        print(\"=\" * 60)\n        print(\"HFT COMPREHENSIVE FACTOR SELECTION\")\n        print(\"=\" * 60)\n        \n        print(f\"🎯 Target: {self.config.min_factors}-{self.config.max_factors} factors for 1-min HFT\")\n        print(f\"📊 Basic model: ≤{self.config.optimal_basic} factors\")\n        print(f\"🚀 Enhanced model: {self.config.optimal_basic+1}-{self.config.optimal_enhanced} factors\")\n        print(f\"🔧 Max correlation: {self.config.factor_correlation_threshold}\")\n        print(f\"📁 Results: {self.config.top_results_file}\")\n        \n        # 阶段1: 基础模型优化（5-8因子）\n        print(\"\\n\" + \"=\" * 60)\n        print(f\"STAGE 1: BASIC MODEL OPTIMIZATION ({self.config.min_factors}-{self.config.optimal_basic} factors)\")\n        print(\"=\" * 60)\n        \n        for k in range(self.config.min_factors, self.config.optimal_basic + 1):\n            print(f\"\\n🔍 Searching optimal {k}-factor combinations...\")\n            self.smart_exhaustive_search(X, y, k)\n        \n        # 阶段2: 增强模型优化（9-12因子）\n        if self.config.max_factors > self.config.optimal_basic:\n            print(\"\\n\" + \"=\" * 60)\n            print(f\"STAGE 2: ENHANCED MODEL OPTIMIZATION ({self.config.optimal_basic+1}-{self.config.optimal_enhanced} factors)\")\n            print(\"=\" * 60)\n            \n            # 基于最佳基础模型扩展\n            if len(self.results_manager.results_df) > 0:\n                # 获取最佳基础模型\n                basic_models = self.results_manager.results_df[\n                    self.results_manager.results_df['num_factors'] <= self.config.optimal_basic\n                ].head(5)\n                \n                for idx, row in basic_models.iterrows():\n                    base_factors = row['factors'].split(',')\n                    print(f\"\\n📈 Extending model: {base_factors} (score: {row['score']:.6f})\")\n                    \n                    enhanced_factors, enhanced_score = self.hft_forward_selection(X, y, base_factors)\n                    print(f\"🏆 Enhanced result: {len(enhanced_factors)} factors, score: {enhanced_score:.6f}\")\n        \n        # 阶段3: 复杂模型探索（13-15因子）\n        if self.config.max_factors > self.config.optimal_enhanced:\n            print(\"\\n\" + \"=\" * 60)\n            print(f\"STAGE 3: COMPLEX MODEL EXPLORATION ({self.config.optimal_enhanced+1}-{self.config.max_factors} factors)\")\n            print(\"=\" * 60)\n            \n            # 随机重启搜索\n            for restart in range(3):\n                print(f\"\\n🔄 Random restart {restart + 1}/3\")\n                \n                # 从不同tier组合开始\n                np.random.seed(self.config.seed + restart)\n                tier1_factors = self.factor_tiers.get(\"tier1\", [])\n                tier2_factors = self.factor_tiers.get(\"tier2\", [])\n                \n                if len(tier1_factors) >= 4 and len(tier2_factors) >= 2:\n                    start_factors = (\n                        np.random.choice(tier1_factors, 4, replace=False).tolist() +\n                        np.random.choice(tier2_factors, 2, replace=False).tolist()\n                    )\n                else:\n                    start_factors = np.random.choice(\n                        self.candidate_factors[:25], \n                        self.config.min_factors + 1,\n                        replace=False\n                    ).tolist()\n                \n                complex_factors, complex_score = self.hft_forward_selection(X, y, start_factors)\n                print(f\"🎯 Complex result: {len(complex_factors)} factors, score: {complex_score:.6f}\")\n        \n        # 最终结果分析\n        print(\"\\n\" + \"=\" * 60)\n        print(\"HFT FACTOR SELECTION COMPLETED\")\n        print(\"=\" * 60)\n        \n        print(self.results_manager.get_hft_summary())\n        \n        # 生成推荐方案\n        self._generate_hft_recommendations()\n    \n    def _generate_hft_recommendations(self):\n        \"\"\"生成高频交易推荐方案\"\"\"\n        if len(self.results_manager.results_df) == 0:\n            print(\"No results to generate recommendations.\")\n            return\n        \n        print(\"\\n\" + \"=\" * 60)\n        print(\"HFT DEPLOYMENT RECOMMENDATIONS\")\n        print(\"=\" * 60)\n        \n        # 基础模型推荐\n        basic_models = self.results_manager.results_df[\n            self.results_manager.results_df['num_factors'] <= self.config.optimal_basic\n        ].head(3)\n        \n        print(f\"\\n🏆 TOP 3 BASIC MODELS (≤{self.config.optimal_basic} factors):\")\n        print(\"   Recommended for: Live trading, low latency requirements\")\n        for i, (_, row) in enumerate(basic_models.iterrows(), 1):\n            print(f\"   {i}. Score: {row['score']:.6f}, HFT Score: {row['hft_score']:.6f}\")\n            print(f\"      Factors ({row['num_factors']}): {row['factors']}\")\n            print(f\"      Avg Correlation: {row['avg_correlation']:.3f}\")\n        \n        # 增强模型推荐\n        enhanced_models = self.results_manager.results_df[\n            (self.results_manager.results_df['num_factors'] > self.config.optimal_basic) &\n            (self.results_manager.results_df['num_factors'] <= self.config.optimal_enhanced)\n        ].head(2)\n        \n        if len(enhanced_models) > 0:\n            print(f\"\\n🚀 TOP 2 ENHANCED MODELS ({self.config.optimal_basic+1}-{self.config.optimal_enhanced} factors):\")\n            print(\"   Recommended for: Research, backtesting, GPU-accelerated trading\")\n            for i, (_, row) in enumerate(enhanced_models.iterrows(), 1):\n                print(f\"   {i}. Score: {row['score']:.6f}, HFT Score: {row['hft_score']:.6f}\")\n                print(f\"      Factors ({row['num_factors']}): {row['factors'][:80]}...\")\n                print(f\"      Avg Correlation: {row['avg_correlation']:.3f}\")\n        \n        print(f\"\\n💡 DEPLOYMENT TIPS:\")\n        print(f\"   • Use basic models for production trading\")\n        print(f\"   • Monitor factor correlation < {self.config.factor_correlation_threshold}\")\n        print(f\"   • Update factor weights daily\")\n        print(f\"   • Re-evaluate factor pool monthly\")\n\nclass DataProcessor:\n    \"\"\"数据处理器（HFT优化版）\"\"\"\n    \n    def __init__(self, config: Config):\n        self.config = config\n\n    def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:\n        \"\"\"高频交易数据清理\"\"\"\n        # 处理无穷值和NaN\n        df = df.replace([np.inf, -np.inf], np.nan)\n        \n        # 高频数据常用的前向填充\n        df = df.fillna(method='ffill').fillna(method='bfill').fillna(0)\n        \n        return df\n    \n    def load_and_process_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n        \"\"\"加载和处理数据\"\"\"\n        print(\"Loading data for HFT factor selection...\")\n        X_train = pd.read_parquet(self.config.X_train_path)\n        y_train = pd.read_parquet(self.config.y_train_path)\n        if isinstance(y_train, pd.DataFrame):\n            y_train = y_train.iloc[:, 0]\n        X_test = pd.read_parquet(self.config.X_test_path)\n        \n        # 数据清理\n        X_train = self.clean_data(X_train)\n        X_test = self.clean_data(X_test)\n        \n        # 确保列名格式正确\n        X_train.columns = [f'X{i}' for i in range(X_train.shape[1])]\n        X_test.columns = [f'X{i}' for i in range(X_test.shape[1])]\n        \n        print(f\"Training data shape: {X_train.shape}\")\n        print(f\"Test data shape: {X_test.shape}\")\n        print(f\"Number of features: {X_train.shape[1]}\")\n        \n        return X_train, y_train, X_test\n\ndef main():\n    \"\"\"主执行函数\"\"\"\n    print(\"=\" * 60)\n    print(\"HIGH-FREQUENCY TRADING FACTOR SELECTION\")\n    print(\"=\" * 60)\n    \n    # 初始化配置\n    config = Config()\n    print(f\"✅ Configuration initialized for 1-minute HFT\")\n    print(f\"🎯 Factor range: {config.min_factors}-{config.max_factors}\")\n    print(f\"🔧 Correlation threshold: {config.factor_correlation_threshold}\")\n    print(f\"💻 GPU enabled: {config.use_gpu}\")\n    print(f\"📊 Single factor file: {config.single_factor_file}\")\n    \n    # 加载数据\n    data_processor = DataProcessor(config)\n    X_train, y_train, X_test = data_processor.load_and_process_data()\n    \n    # 使用样本数据进行高效搜索\n    sample_size = min(config.sample_size_for_search, len(X_train))\n    np.random.seed(config.seed)\n    sample_indices = np.random.choice(len(X_train), sample_size, replace=False)\n    X_sample = X_train.iloc[sample_indices]\n    y_sample = y_train.iloc[sample_indices]\n    \n    print(f\"🔍 Using {sample_size:,} samples for factor search\")\n    print(f\"📈 Time range: {X_sample.index.min()} to {X_sample.index.max()}\")\n    \n    # 运行HFT因子选择\n    factor_selector = HFTGreedyFactorSelector(config)\n    factor_selector.run_hft_comprehensive_search(X_sample, y_sample)\n    \n    print(f\"\\n🎉 HFT factor selection completed!\")\n    print(f\"📁 Results saved to: {config.top_results_file}\")\n    print(f\"🚀 Ready for high-frequency trading deployment!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"execution_failed":"2025-06-04T09:28:25.781Z"}},"outputs":[],"execution_count":null}]}