{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom xgboost import XGBRegressor\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom sklearn.feature_selection import mutual_info_regression, f_regression, SelectKBest\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet\nfrom scipy.stats import pearsonr, spearmanr, rankdata, ks_2samp, skew, kurtosis\nfrom scipy.optimize import differential_evolution\nimport gc\nimport os\nfrom datetime import datetime\nfrom typing import Dict, List, Tuple, Any, Optional\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nfrom tqdm import tqdm\n\n# Set visualization style\nplt.style.use('default')\nsns.set_palette(\"husl\")\n\nprint(\"=\" * 80)\nprint(\"ENHANCED CRYPTO MARKET PREDICTION WITH TIME SERIES INTELLIGENCE\")\nprint(\"=\" * 80)\nprint(\"\\nKey Improvements:\")\nprint(\"1. Better pseudo-label generation using ensemble methods\")\nprint(\"2. Adaptive feature transformations based on distribution analysis\")\nprint(\"3. Improved stability metrics with multiple validation windows\")\nprint(\"4. Smarter feature selection balancing stability and predictive power\")\nprint(\"5. Model stacking with confidence weighting\")\nprint(\"=\" * 80)\n\n# Configuration\nclass CFG:\n # Data paths\n train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n timestamp_recon_path = '/kaggle/input/the-order-of-the-test-rows-2/closest_rows.csv'\n \n # Model settings\n n_folds = 5\n random_state = 42\n \n # Feature engineering settings\n n_proprietary_features = 40\n n_interaction_features = 60\n n_temporal_features = 30\n n_robust_features = 20 # New: features designed to be distribution-robust\n \n # Time series settings\n lag_periods = [1, 3, 5, 7, 10, 15, 20, 30, 60]\n rolling_windows = [5, 10, 20, 50, 100]\n \n # Important features from analysis\n important_x_features = [\n \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n \"X758\", \"X296\", \"X611\", \"X780\", \"X451\", \"X25\", \"X591\", \"X363\", \"X405\", \"X321\",\n \"X175\", \"X179\", \"X197\", \"X22\", \"X40\", \"X181\", \"X28\", \"X169\", \"X198\", \"X173\"\n ]\n \n # Visualization settings\n figure_dpi = 150\n save_figures = True\n \n # Intelligence settings\n stability_threshold = 0.5\n very_stable_threshold = 0.7\n importance_threshold = 0.01\n drift_threshold = 0.3\n\n# Utility functions\ndef print_section(title):\n \"\"\"Print formatted section header\"\"\"\n print(f\"\\n{'='*60}\")\n print(f\"{title.upper()}\")\n print(f\"{'='*60}\")\n\ndef reduce_mem_usage(df, name=\"\"):\n \"\"\"Optimize memory usage of DataFrame - Fixed for float16 issue\"\"\"\n print(f\"\\nOptimizing memory for {name}...\")\n start_mem = df.memory_usage().sum() / 1024**2\n \n for col in df.columns:\n col_type = df[col].dtype\n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n else:\n # Skip float16 conversion to avoid pandas issues\n if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n \n end_mem = df.memory_usage().sum() / 1024**2\n print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n gc.collect()\n return df\n\n# Enhanced Feature Engineering Functions\ndef add_market_features(df):\n \"\"\"Create comprehensive market microstructure features\"\"\"\n print(\"Engineering market microstructure features...\")\n \n # Basic interactions\n df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-8)\n df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n \n # Pressure indicators\n df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-8)\n df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n \n # Liquidity features\n df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)\n df['liquidity_ratio'] = df['total_liquidity'] / (df['volume'] + 1e-8)\n \n # Volume transformations\n df['log_volume'] = np.log1p(df['volume'])\n df['sqrt_volume'] = np.sqrt(df['volume'])\n df['cbrt_volume'] = np.cbrt(df['volume'])\n \n # Market microstructure metrics\n df['kyle_lambda'] = df['order_flow_imbalance'] / (df['sqrt_volume'] + 1e-8)\n df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n df['amihud_illiquidity'] = np.abs(df['net_pressure']) / (df['log_volume'] + 1e-8)\n \n # Advanced features\n df['effective_spread'] = 2 * np.abs(df['order_flow_imbalance']) * df['bid_ask_spread']\n df['realized_spread'] = df['bid_ask_spread'] * df['vpin']\n df['price_impact'] = df['kyle_lambda'] * df['volume']\n df['trade_intensity'] = df['volume'] / (df['total_liquidity'] + 1e-8)\n \n # Order book imbalance features\n df['book_pressure'] = (df['bid_qty'] * df['buy_qty'] - df['ask_qty'] * df['sell_qty']) / \\\n ((df['bid_qty'] * df['buy_qty'] + df['ask_qty'] * df['sell_qty']) + 1e-8)\n df['depth_ratio'] = (df['bid_qty'] + df['buy_qty']) / (df['ask_qty'] + df['sell_qty'] + 1e-8)\n \n # Toxicity measures\n df['flow_toxicity'] = df['vpin'] * df['sqrt_volume']\n df['adverse_selection'] = df['kyle_lambda'] * df['trade_intensity']\n \n # New advanced features\n df['hasbrouck_lambda'] = df['order_flow_imbalance'] / (np.log1p(df['volume']) + 1e-8)\n df['roll_measure'] = 2 * np.sqrt(np.abs(df['order_flow_imbalance'] * df['bid_ask_spread']))\n df['informed_trading'] = df['vpin'] * df['kyle_lambda']\n \n # Handle infinities and NaN\n df = df.replace([np.inf, -np.inf], np.nan)\n df = df.fillna(0)\n \n return df\n\ndef create_robust_features(df, n_features=20):\n \"\"\"Create features designed to be robust to distribution shifts - Fixed\"\"\"\n print(f\"Creating {n_features} robust features...\")\n \n # Get X features\n x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n base_features = [f for f in CFG.important_x_features if f in df.columns][:25]\n \n robust_idx = 1\n \n # Percentile-based features (robust to outliers) - Fixed to avoid float16 issue\n for i in range(min(5, n_features)):\n if i < len(base_features):\n feat = base_features[i]\n # Convert to float32 before qcut to avoid float16 issues\n feat_values = df[feat].astype(np.float32)\n try:\n df[f'X_robust_{robust_idx}'] = pd.qcut(feat_values, q=10, labels=False, duplicates='drop') / 10.0\n except:\n # Fallback to rank if qcut fails\n df[f'X_robust_{robust_idx}'] = rankdata(feat_values) / len(feat_values)\n robust_idx += 1\n \n # Robust z-scores using median and MAD\n for i in range(min(5, n_features - robust_idx + 1)):\n if i < len(base_features):\n feat = base_features[i]\n median = df[feat].median()\n mad = (df[feat] - median).abs().median()\n df[f'X_robust_{robust_idx}'] = (df[feat] - median) / (mad + 1e-8)\n df[f'X_robust_{robust_idx}'] = df[f'X_robust_{robust_idx}'].clip(-3, 3)\n robust_idx += 1\n \n # Winsorized features\n for i in range(min(5, n_features - robust_idx + 1)):\n if i < len(base_features):\n feat = base_features[i+5]\n lower = df[feat].quantile(0.05)\n upper = df[feat].quantile(0.95)\n df[f'X_robust_{robust_idx}'] = df[feat].clip(lower, upper)\n robust_idx += 1\n \n # Sign-based features\n for i in range(min(5, n_features - robust_idx + 1)):\n if i < len(base_features):\n feat = base_features[i+10]\n df[f'X_robust_{robust_idx}'] = np.sign(df[feat])\n robust_idx += 1\n \n # Handle infinities and NaN\n df = df.replace([np.inf, -np.inf], np.nan)\n df = df.fillna(0)\n \n return df\n\ndef create_temporal_aware_features(df, n_features=30):\n \"\"\"Create features designed to be robust across time periods\"\"\"\n print(f\"Creating {n_features} temporal-aware features...\")\n \n # Get X features\n x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n \n # Prioritize important features\n base_features = [f for f in CFG.important_x_features if f in df.columns][:20]\n \n # Add available features if needed\n if len(base_features) < 10:\n for feat in x_features[:20]:\n if feat not in base_features:\n base_features.append(feat)\n \n temp_idx = 1\n \n # Normalized rank features (robust to distribution shifts)\n for i in range(min(10, n_features, len(base_features))):\n feat = base_features[i]\n df[f'X_temp_{temp_idx}'] = rankdata(df[feat]) / len(df)\n temp_idx += 1\n \n # Robust statistical features\n if len(base_features) >= 5:\n # Robust mean (trimmed mean)\n selected_cols = df[base_features[:5]].copy()\n lower_bound = selected_cols.quantile(0.1)\n upper_bound = selected_cols.quantile(0.9)\n \n # Clip each column\n for col in selected_cols.columns:\n selected_cols[col] = selected_cols[col].clip(lower_bound[col], upper_bound[col])\n \n df[f'X_temp_{temp_idx}'] = selected_cols.mean(axis=1)\n temp_idx += 1\n \n # Median absolute deviation\n median_vals = df[base_features[:5]].median(axis=1)\n df[f'X_temp_{temp_idx}'] = df[base_features[:5]].sub(median_vals, axis=0).abs().median(axis=1)\n temp_idx += 1\n \n # Interquartile range features\n df[f'X_temp_{temp_idx}'] = df[base_features[:7]].quantile(0.75, axis=1) - \\\n df[base_features[:7]].quantile(0.25, axis=1)\n temp_idx += 1\n \n # Relative features (compared to rolling statistics)\n for window in [20, 50]:\n if temp_idx >= n_features:\n break\n for i, feat in enumerate(base_features[:5]):\n if temp_idx >= n_features:\n break\n roll_mean = df[feat].rolling(window, min_periods=1).mean()\n roll_std = df[feat].rolling(window, min_periods=1).std() + 1e-8\n df[f'X_temp_{temp_idx}'] = (df[feat] - roll_mean) / roll_std\n temp_idx += 1\n \n # Cross-sectional features\n for i in range(0, min(len(base_features)-1, 10), 2):\n if temp_idx >= n_features:\n break\n feat1, feat2 = base_features[i], base_features[i+1]\n # Normalized difference\n df[f'X_temp_{temp_idx}'] = (df[feat1] - df[feat2]) / (df[feat1].std() + df[feat2].std() + 1e-8)\n temp_idx += 1\n \n # Quantile-based features\n while temp_idx < n_features and len(base_features) >= 3:\n feat_idx = (temp_idx - 1) % len(base_features)\n feat = base_features[feat_idx]\n quantile_val = 0.1 + (0.8 * feat_idx / len(base_features))\n df[f'X_temp_{temp_idx}'] = (df[feat] > df[feat].quantile(quantile_val)).astype(int)\n temp_idx += 1\n \n # Handle infinities and NaN\n df = df.replace([np.inf, -np.inf], np.nan)\n df = df.fillna(0)\n \n return df\n\ndef create_proprietary_features(df, n_features=30):\n \"\"\"Create proprietary X variable combinations\"\"\"\n print(f\"Creating {n_features} proprietary features...\")\n \n # Get existing X features\n x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n \n # Prioritize important features\n base_features = [f for f in CFG.important_x_features if f in df.columns][:15]\n \n # Add high-variance features if needed\n if len(base_features) < 15:\n x_variances = df[x_features].var()\n high_var_features = x_variances.nlargest(15).index.tolist()\n for feat in high_var_features:\n if feat not in base_features:\n base_features.append(feat)\n if len(base_features) >= 15:\n break\n \n prop_idx = 1\n \n # Statistical combinations\n if len(base_features) >= 5:\n df[f'X_prop_{prop_idx}'] = df[base_features[:5]].mean(axis=1)\n prop_idx += 1\n df[f'X_prop_{prop_idx}'] = df[base_features[:5]].std(axis=1)\n prop_idx += 1\n df[f'X_prop_{prop_idx}'] = df[base_features[:5]].max(axis=1) - df[base_features[:5]].min(axis=1)\n prop_idx += 1\n df[f'X_prop_{prop_idx}'] = df[base_features[:5]].median(axis=1)\n prop_idx += 1\n df[f'X_prop_{prop_idx}'] = df[base_features[:7]].quantile(0.25, axis=1)\n prop_idx += 1\n df[f'X_prop_{prop_idx}'] = df[base_features[:7]].quantile(0.75, axis=1)\n prop_idx += 1\n \n # Skewness and kurtosis\n df[f'X_prop_{prop_idx}'] = df[base_features[:10]].skew(axis=1)\n prop_idx += 1\n df[f'X_prop_{prop_idx}'] = df[base_features[:10]].kurtosis(axis=1)\n prop_idx += 1\n \n # Non-linear transformations\n for i in range(min(10, n_features - prop_idx + 1)):\n if prop_idx > n_features:\n break\n feat = base_features[i % len(base_features)]\n if i < 3:\n df[f'X_prop_{prop_idx}'] = np.sign(df[feat]) * np.sqrt(np.abs(df[feat]))\n elif i < 6:\n df[f'X_prop_{prop_idx}'] = np.tanh(df[feat] / (df[feat].std() + 1e-8))\n else:\n df[f'X_prop_{prop_idx}'] = rankdata(df[feat]) / len(df)\n prop_idx += 1\n \n # Market interactions\n if 'volume' in df.columns:\n for i in range(min(5, n_features - prop_idx + 1)):\n if prop_idx > n_features or i >= len(base_features):\n break\n df[f'X_prop_{prop_idx}'] = df[base_features[i]] * np.log1p(df['volume'])\n prop_idx += 1\n \n # Feature ratios\n while prop_idx <= n_features and len(base_features) > 1:\n i = (prop_idx - 1) % (len(base_features) - 1)\n feat1 = base_features[i]\n feat2 = base_features[i + 1]\n df[f'X_prop_{prop_idx}'] = df[feat1] / (np.abs(df[feat2]) + 1e-8)\n prop_idx += 1\n \n # Handle infinities and NaN\n df = df.replace([np.inf, -np.inf], np.nan)\n df = df.fillna(0)\n \n return df\n\ndef create_interaction_features(df, selected_features, n_interactions=50):\n \"\"\"Create interaction features between important variables\"\"\"\n print(f\"Creating {n_interactions} interaction features...\")\n \n interaction_features = []\n feature_names = []\n \n # Priority features\n important_x = [f for f in CFG.important_x_features if f in selected_features and f in df.columns][:10]\n market_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'liquidity_imbalance',\n 'bid_ask_spread', 'buying_pressure', 'volume', 'log_volume',\n 'amihud_illiquidity', 'flow_toxicity']\n priority_market = [f for f in market_features if f in selected_features and f in df.columns][:8]\n \n interaction_count = 0\n \n # X features with market microstructure\n for i, x_feat in enumerate(important_x[:10]):\n if interaction_count >= n_interactions // 2:\n break\n for j, market_feat in enumerate(priority_market[:5]):\n if interaction_count >= n_interactions // 2:\n break\n \n interaction_features.append(df[x_feat] * df[market_feat])\n feature_names.append(f'{x_feat}_x_{market_feat}')\n interaction_count += 1\n \n # Polynomial features for important X\n for i in range(min(10, len(important_x))):\n if interaction_count >= n_interactions:\n break\n feat = important_x[i]\n \n interaction_features.append(df[feat] ** 2)\n feature_names.append(f'{feat}_squared')\n interaction_count += 1\n \n # Market feature interactions\n market_pairs = [\n ('order_flow_imbalance', 'kyle_lambda'),\n ('vpin', 'liquidity_imbalance'),\n ('buying_pressure', 'selling_pressure'),\n ('bid_ask_spread', 'total_liquidity'),\n ('flow_toxicity', 'trade_intensity'),\n ('amihud_illiquidity', 'volume')\n ]\n \n for feat1, feat2 in market_pairs:\n if interaction_count >= n_interactions:\n break\n if feat1 in df.columns and feat2 in df.columns:\n interaction_features.append(df[feat1] * df[feat2])\n feature_names.append(f'{feat1}_x_{feat2}')\n interaction_count += 1\n \n # Create DataFrame\n if interaction_features:\n interaction_df = pd.DataFrame(\n np.column_stack(interaction_features[:interaction_count]),\n columns=feature_names[:interaction_count],\n index=df.index\n )\n \n # Handle infinities and NaN\n interaction_df = interaction_df.replace([np.inf, -np.inf], np.nan)\n interaction_df = interaction_df.fillna(0)\n else:\n interaction_df = pd.DataFrame(index=df.index)\n \n print(f\"Created {interaction_count} interaction features\")\n return interaction_df\n\n# Enhanced Time Series Intelligence Extractor\nclass EnhancedTimeSeriesIntelligenceExtractor:\n \"\"\"\n Enhanced version with better pseudo-labels and stability analysis\n \"\"\"\n \n def __init__(self):\n self.feature_stability = None\n self.temporal_patterns = None\n self.train_importances = None\n self.test_importances = None\n self.intelligence_report = {}\n self.feature_drift_scores = {}\n self.multi_window_stability = {}\n \n def analyze_with_time_series(self, train_df, test_df_ordered, y_train, selected_features):\n \"\"\"Extract comprehensive intelligence from time series structure\"\"\"\n print(\"\\n\" + \"=\"*50)\n print(\"EXTRACTING ENHANCED TIME SERIES INTELLIGENCE\")\n print(\"=\"*50)\n \n # 1. Multi-window stability analysis\n print(\"\\n1. Analyzing feature stability across multiple time windows...\")\n self.analyze_multi_window_stability(train_df, test_df_ordered, selected_features)\n \n # 2. Create better synthetic labels\n print(\"\\n2. Creating enhanced synthetic labels using ensemble...\")\n test_predictions = self.create_enhanced_pseudo_labels(train_df, test_df_ordered, y_train)\n \n # 3. Compare feature importance with better models\n print(\"\\n3. Comparing feature importance with improved models...\")\n self.analyze_feature_importance_enhanced(\n train_df, test_df_ordered, y_train, test_predictions, selected_features\n )\n \n # 4. Analyze temporal drift with autocorrelation\n print(\"\\n4. Analyzing temporal drift and autocorrelation patterns...\")\n self.analyze_temporal_drift_enhanced(train_df, test_df_ordered, selected_features)\n \n # 5. Feature distribution analysis\n print(\"\\n5. Analyzing feature distributions...\")\n self.analyze_feature_distributions(train_df, test_df_ordered, selected_features)\n \n # 6. Generate enhanced intelligence report\n print(\"\\n6. Generating enhanced intelligence report...\")\n self.generate_enhanced_intelligence_report()\n \n return self.intelligence_report\n \n def create_enhanced_pseudo_labels(self, train_df, test_df, y_train):\n \"\"\"Create better pseudo-labels using ensemble\"\"\"\n # Use multiple simple models\n models = [\n Ridge(alpha=1.0),\n Lasso(alpha=0.1),\n ElasticNet(alpha=0.1),\n RandomForestRegressor(n_estimators=100, max_depth=6, random_state=42, n_jobs=-1),\n XGBRegressor(n_estimators=100, max_depth=4, learning_rate=0.1, random_state=42, verbosity=0)\n ]\n \n # Use stable features for pseudo-labeling\n stable_features = ['volume', 'buy_qty', 'sell_qty', 'bid_qty', 'ask_qty',\n 'order_flow_imbalance', 'kyle_lambda', 'vpin']\n available_features = [f for f in stable_features if f in train_df.columns and f in test_df.columns]\n \n X_train = train_df[available_features]\n X_test = test_df[available_features]\n \n predictions = []\n for model in models:\n model.fit(X_train, y_train)\n pred = model.predict(X_test)\n predictions.append(pred)\n \n # Weighted average with more weight on tree-based models\n weights = [0.1, 0.1, 0.1, 0.35, 0.35] # More weight on RF and XGB\n ensemble_pred = np.average(predictions, axis=0, weights=weights)\n \n return ensemble_pred\n \n def analyze_multi_window_stability(self, train_df, test_df, feature_cols):\n \"\"\"Analyze stability across multiple validation windows\"\"\"\n stability_metrics = {}\n \n # Split train data into multiple windows\n windows = [0.5, 0.7, 0.9] # Use last 50%, 70%, 90% of train data\n \n for col in tqdm(feature_cols, desc=\"Analyzing stability\"):\n if col in train_df.columns and col in test_df.columns:\n window_stabilities = []\n \n for window in windows:\n start_idx = int(len(train_df) * (1 - window))\n train_window = train_df[col].iloc[start_idx:]\n \n # Calculate stability metrics\n train_mean = train_window.mean()\n test_mean = test_df[col].mean()\n train_std = train_window.std() + 1e-8\n test_std = test_df[col].std() + 1e-8\n \n mean_shift = abs(train_mean - test_mean) / train_std\n std_ratio = test_std / train_std\n \n # KS test\n try:\n train_sample = train_window.sample(min(5000, len(train_window)), random_state=42)\n test_sample = test_df[col].sample(min(5000, len(test_df)), random_state=42)\n ks_stat, ks_pval = ks_2samp(train_sample, test_sample)\n except:\n ks_stat, ks_pval = 1.0, 0.0\n \n stability_score = 1 / (1 + mean_shift + abs(1 - std_ratio) + ks_stat)\n window_stabilities.append(stability_score)\n \n # Overall stability is minimum across windows (most conservative)\n overall_stability = min(window_stabilities)\n stability_trend = window_stabilities[-1] - window_stabilities[0] # Positive means improving\n \n stability_metrics[col] = {\n 'stability_scores': window_stabilities,\n 'overall_stability': overall_stability,\n 'stability_trend': stability_trend,\n 'is_stable': overall_stability > CFG.stability_threshold,\n 'is_very_stable': overall_stability > CFG.very_stable_threshold,\n 'is_improving': stability_trend > 0.1\n }\n \n self.multi_window_stability = pd.DataFrame(stability_metrics).T\n self.feature_stability = self.multi_window_stability # For compatibility\n \n def analyze_feature_importance_enhanced(self, train_df, test_df, y_train, y_test_pred, selected_features):\n \"\"\"Enhanced importance analysis with better models\"\"\"\n valid_features = [f for f in selected_features if f in train_df.columns and f in test_df.columns]\n \n # Use better hyperparameters\n model_params = {\n 'n_estimators': 300,\n 'max_depth': 8,\n 'learning_rate': 0.03,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'min_child_weight': 3,\n 'reg_alpha': 1.0,\n 'reg_lambda': 1.0,\n 'random_state': 42,\n 'verbosity': 0\n }\n \n # 1. Train-only model\n print(\" Training enhanced model on train data...\")\n xgb_train = XGBRegressor(**model_params)\n xgb_train.fit(train_df[valid_features], y_train)\n self.train_importances = pd.Series(\n xgb_train.feature_importances_, \n index=valid_features\n ).sort_values(ascending=False)\n \n # 2. Test-only model with pseudo-labels\n print(\" Training model on test data with enhanced pseudo-labels...\")\n xgb_test = XGBRegressor(**model_params)\n xgb_test.fit(test_df[valid_features], y_test_pred)\n self.test_importances = pd.Series(\n xgb_test.feature_importances_, \n index=valid_features\n ).sort_values(ascending=False)\n \n # 3. Time-weighted combined model\n print(\" Training time-weighted combined model...\")\n # Give more weight to recent data\n time_weights = np.concatenate([\n np.linspace(0.5, 1.0, len(y_train)),\n np.ones(len(y_test_pred))\n ])\n \n combined_X = pd.concat([train_df[valid_features], test_df[valid_features]], ignore_index=True)\n combined_y = np.concatenate([y_train, y_test_pred])\n \n xgb_combined = XGBRegressor(**model_params)\n xgb_combined.fit(combined_X, combined_y, sample_weight=time_weights)\n self.combined_importances = pd.Series(\n xgb_combined.feature_importances_, \n index=valid_features\n ).sort_values(ascending=False)\n \n def analyze_temporal_drift_enhanced(self, train_df, test_df, feature_cols):\n \"\"\"Enhanced temporal drift analysis\"\"\"\n drift_scores = {}\n \n # Multiple split points for robustness\n split_points = [0.5, 0.7, 0.9]\n \n for col in tqdm(feature_cols, desc=\"Analyzing drift\"):\n if col in train_df.columns:\n try:\n drift_metrics = []\n \n for split in split_points:\n split_idx = int(len(train_df) * split)\n train_early = train_df[col][:split_idx]\n train_late = train_df[col][split_idx:]\n \n early_mean = train_early.mean()\n late_mean = train_late.mean()\n train_std = train_df[col].std() + 1e-8\n \n drift = abs(early_mean - late_mean) / train_std\n drift_metrics.append(drift)\n \n # Trend analysis\n x = np.arange(len(train_df))\n y = train_df[col].values\n trend_coef = np.polyfit(x, y, 1)[0]\n trend_strength = abs(trend_coef) * len(train_df) / train_std\n \n # Enhanced autocorrelation\n autocorr_lags = [1, 5, 10, 20]\n autocorrs = []\n for lag in autocorr_lags:\n try:\n autocorrs.append(train_df[col].autocorr(lag=lag))\n except:\n autocorrs.append(0)\n \n # Check for seasonality (simple FFT-based)\n try:\n fft = np.fft.fft(train_df[col].values)\n power = np.abs(fft) ** 2\n freqs = np.fft.fftfreq(len(train_df))\n # Find dominant frequency (exclude DC component)\n dominant_freq_idx = np.argmax(power[1:len(power)//2]) + 1\n seasonality_strength = power[dominant_freq_idx] / np.sum(power)\n except:\n seasonality_strength = 0\n \n drift_scores[col] = {\n 'drift_scores': drift_metrics,\n 'max_drift': max(drift_metrics),\n 'drift_trend': drift_metrics[-1] - drift_metrics[0],\n 'trend_strength': trend_strength,\n 'autocorr_1': autocorrs[0] if autocorrs else 0,\n 'autocorr_mean': np.mean(autocorrs) if autocorrs else 0,\n 'seasonality': seasonality_strength,\n 'has_drift': max(drift_metrics) > CFG.drift_threshold,\n 'has_trend': trend_strength > 0.5,\n 'has_memory': abs(autocorrs[0]) > 0.5 if autocorrs else False\n }\n except:\n pass\n \n self.feature_drift_scores = pd.DataFrame(drift_scores).T\n \n def analyze_feature_distributions(self, train_df, test_df, feature_cols):\n \"\"\"Analyze distribution characteristics\"\"\"\n distribution_metrics = {}\n \n for col in feature_cols:\n if col in train_df.columns and col in test_df.columns:\n try:\n train_vals = train_df[col]\n test_vals = test_df[col]\n \n # Distribution shape metrics\n train_skew = skew(train_vals)\n test_skew = skew(test_vals)\n train_kurt = kurtosis(train_vals)\n test_kurt = kurtosis(test_vals)\n \n # Outlier analysis\n train_q1, train_q3 = train_vals.quantile([0.25, 0.75])\n train_iqr = train_q3 - train_q1\n train_outliers = ((train_vals < train_q1 - 1.5*train_iqr) | \n (train_vals > train_q3 + 1.5*train_iqr)).sum() / len(train_vals)\n \n test_q1, test_q3 = test_vals.quantile([0.25, 0.75])\n test_iqr = test_q3 - test_q1\n test_outliers = ((test_vals < test_q1 - 1.5*test_iqr) | \n (test_vals > test_q3 + 1.5*test_iqr)).sum() / len(test_vals)\n \n distribution_metrics[col] = {\n 'train_skew': train_skew,\n 'test_skew': test_skew,\n 'skew_change': abs(test_skew - train_skew),\n 'train_kurt': train_kurt,\n 'test_kurt': test_kurt,\n 'kurt_change': abs(test_kurt - train_kurt),\n 'train_outliers': train_outliers,\n 'test_outliers': test_outliers,\n 'is_heavy_tailed': train_kurt > 3 or test_kurt > 3,\n 'is_skewed': abs(train_skew) > 1 or abs(test_skew) > 1,\n 'needs_transform': (abs(train_skew) > 2 or train_outliers > 0.05)\n }\n except:\n pass\n \n self.distribution_metrics = pd.DataFrame(distribution_metrics).T\n \n def generate_enhanced_intelligence_report(self):\n \"\"\"Generate comprehensive intelligence report with better insights\"\"\"\n self.intelligence_report = {\n 'stable_features': [],\n 'very_stable_features': [],\n 'unstable_features': [],\n 'improving_features': [],\n 'robust_features': [],\n 'features_to_remove': [],\n 'features_to_regularize': [],\n 'features_to_transform': [],\n 'drifting_features': [],\n 'trending_features': [],\n 'memory_features': [],\n 'seasonal_features': [],\n 'engineering_insights': [],\n 'model_recommendations': [],\n 'transformation_recommendations': {}\n }\n \n # Process stability analysis\n if self.multi_window_stability is not None:\n stable_features = self.multi_window_stability[\n self.multi_window_stability['is_stable']\n ].index.tolist()\n self.intelligence_report['stable_features'] = stable_features[:100]\n \n very_stable_features = self.multi_window_stability[\n self.multi_window_stability['is_very_stable']\n ].index.tolist()\n self.intelligence_report['very_stable_features'] = very_stable_features[:50]\n \n unstable_features = self.multi_window_stability[\n self.multi_window_stability['overall_stability'] < CFG.stability_threshold\n ].index.tolist()\n self.intelligence_report['unstable_features'] = unstable_features[:50]\n \n improving_features = self.multi_window_stability[\n self.multi_window_stability['is_improving']\n ].index.tolist()\n self.intelligence_report['improving_features'] = improving_features[:30]\n \n # Enhanced feature categorization\n if self.train_importances is not None and self.test_importances is not None:\n robust_features = []\n features_to_remove = []\n features_to_regularize = []\n \n # Calculate importance consistency\n for feat in self.train_importances.index:\n if feat in self.test_importances.index and feat in self.multi_window_stability.index:\n train_imp = self.train_importances[feat]\n test_imp = self.test_importances[feat]\n stability = self.multi_window_stability.loc[feat, 'overall_stability']\n \n # Importance ratio\n imp_ratio = min(train_imp, test_imp) / (max(train_imp, test_imp) + 1e-8)\n \n # Robust features: consistently important and stable\n if (train_imp > CFG.importance_threshold and \n test_imp > CFG.importance_threshold and \n stability > CFG.very_stable_threshold and\n imp_ratio > 0.5):\n robust_features.append(feat)\n \n # Features to remove: low importance and unstable\n elif ((train_imp < 0.005 and test_imp < 0.005) or\n (stability < 0.3 and test_imp < 0.01) or\n (train_imp > 0.05 and test_imp < 0.001)):\n features_to_remove.append(feat)\n \n # Features to regularize: important but unstable\n elif (train_imp > 0.02 and stability < CFG.stability_threshold):\n features_to_regularize.append(feat)\n \n self.intelligence_report['robust_features'] = robust_features[:50]\n self.intelligence_report['features_to_remove'] = features_to_remove[:30]\n self.intelligence_report['features_to_regularize'] = features_to_regularize[:40]\n \n # Process drift analysis\n if self.feature_drift_scores is not None and len(self.feature_drift_scores) > 0:\n drifting_features = self.feature_drift_scores[\n self.feature_drift_scores['has_drift']\n ].index.tolist()\n self.intelligence_report['drifting_features'] = drifting_features[:30]\n \n trending_features = self.feature_drift_scores[\n self.feature_drift_scores['has_trend']\n ].index.tolist()\n self.intelligence_report['trending_features'] = trending_features[:20]\n \n memory_features = self.feature_drift_scores[\n self.feature_drift_scores['has_memory']\n ].index.tolist()\n self.intelligence_report['memory_features'] = memory_features[:30]\n \n seasonal_features = self.feature_drift_scores[\n self.feature_drift_scores['seasonality'] > 0.1\n ].index.tolist()\n self.intelligence_report['seasonal_features'] = seasonal_features[:20]\n \n # Process distribution analysis\n if hasattr(self, 'distribution_metrics') and self.distribution_metrics is not None:\n features_to_transform = self.distribution_metrics[\n self.distribution_metrics['needs_transform']\n ].index.tolist()\n self.intelligence_report['features_to_transform'] = features_to_transform[:30]\n \n # Transformation recommendations\n for feat in features_to_transform[:30]:\n metrics = self.distribution_metrics.loc[feat]\n if metrics['is_skewed'] and metrics['train_skew'] > 0:\n self.intelligence_report['transformation_recommendations'][feat] = 'log'\n elif metrics['is_skewed'] and metrics['train_skew'] < 0:\n self.intelligence_report['transformation_recommendations'][feat] = 'sqrt_neg'\n elif metrics['is_heavy_tailed']:\n self.intelligence_report['transformation_recommendations'][feat] = 'rank'\n else:\n self.intelligence_report['transformation_recommendations'][feat] = 'robust_scale'\n \n # Enhanced insights\n self.intelligence_report['engineering_insights'] = [\n f\"Found {len(self.intelligence_report['robust_features'])} robust features with consistent importance\",\n f\"Identified {len(self.intelligence_report['improving_features'])} features with improving stability\",\n f\"Detected {len(self.intelligence_report['seasonal_features'])} features with seasonal patterns\",\n \"Use ensemble pseudo-labels for better test set analysis\",\n \"Apply feature-specific transformations based on distribution analysis\",\n \"Weight recent data more heavily in time-sensitive models\",\n \"Consider separate models for features with different stability profiles\",\n \"Use adaptive learning rates based on feature drift scores\",\n f\"Transform {len(self.intelligence_report['features_to_transform'])} features with distribution issues\",\n \"Implement online learning for features with strong temporal trends\"\n ]\n \n self.intelligence_report['model_recommendations'] = [\n f\"Remove {len(self.intelligence_report['features_to_remove'])} low-value unstable features\",\n f\"Apply L2 regularization (alpha=2-5) to {len(self.intelligence_report['features_to_regularize'])} unstable features\",\n \"Use QuantileTransformer for heavy-tailed features\",\n \"Implement time-decay sample weights (exponential decay with half-life)\",\n \"Consider ensemble with different regularization strengths\",\n \"Use early stopping with patience=100 for better generalization\",\n \"Apply different max_depth for stable (8-10) vs unstable (4-6) features\",\n \"Use Huber loss for features with high outlier ratios\",\n \"Implement feature-specific learning rates using column sampling\",\n \"Consider adversarial validation for final feature selection\"\n ]\n \n return self.intelligence_report\n \n def visualize_enhanced_intelligence(self, top_n=20):\n \"\"\"Create enhanced visualizations\"\"\"\n print(\"\\nCreating enhanced intelligence visualizations...\")\n \n # Create figure with better layout\n fig = plt.figure(figsize=(24, 16))\n gs = fig.add_gridspec(4, 3, hspace=0.3, wspace=0.3)\n \n # 1. Multi-window stability trends\n ax1 = fig.add_subplot(gs[0, 0])\n if self.multi_window_stability is not None:\n top_features = self.multi_window_stability.nlargest(15, 'overall_stability')\n stability_data = []\n for feat in top_features.index:\n stability_data.append(top_features.loc[feat, 'stability_scores'])\n \n stability_array = np.array(stability_data)\n windows = ['50%', '70%', '90%']\n \n for i, feat in enumerate(top_features.index):\n ax1.plot(windows, stability_array[i], marker='o', label=feat if i < 5 else '')\n \n ax1.set_xlabel('Training Window')\n ax1.set_ylabel('Stability Score')\n ax1.set_title('Stability Trends Across Windows (Top 15)')\n ax1.legend(fontsize=8)\n ax1.grid(True, alpha=0.3)\n \n # 2. Feature importance consistency\n ax2 = fig.add_subplot(gs[0, 1])\n if self.train_importances is not None and self.test_importances is not None:\n common_features = list(set(self.train_importances.index) & set(self.test_importances.index))[:30]\n if common_features:\n train_vals = self.train_importances[common_features]\n test_vals = self.test_importances[common_features]\n \n # Calculate consistency ratio\n consistency = []\n for feat in common_features:\n ratio = min(train_vals[feat], test_vals[feat]) / (max(train_vals[feat], test_vals[feat]) + 1e-8)\n consistency.append(ratio)\n \n scatter = ax2.scatter(train_vals, test_vals, c=consistency, \n cmap='RdYlGn', s=100, alpha=0.7)\n ax2.plot([0, max(train_vals.max(), test_vals.max())], \n [0, max(train_vals.max(), test_vals.max())], \n 'k--', alpha=0.5)\n \n ax2.set_xlabel('Train Importance')\n ax2.set_ylabel('Test Importance')\n ax2.set_title('Feature Importance Consistency')\n plt.colorbar(scatter, ax=ax2, label='Consistency Ratio')\n \n # 3. Drift analysis heatmap\n ax3 = fig.add_subplot(gs[0, 2])\n if self.feature_drift_scores is not None and len(self.feature_drift_scores) > 0:\n drift_features = self.feature_drift_scores.nlargest(20, 'max_drift')\n drift_matrix = []\n for feat in drift_features.index:\n drift_matrix.append(drift_features.loc[feat, 'drift_scores'])\n \n drift_array = np.array(drift_matrix)\n im = ax3.imshow(drift_array, aspect='auto', cmap='Reds')\n ax3.set_yticks(range(len(drift_features)))\n ax3.set_yticklabels(drift_features.index, fontsize=8)\n ax3.set_xticks(range(len(drift_matrix[0])))\n ax3.set_xticklabels(['50%', '70%', '90%'])\n ax3.set_xlabel('Split Point')\n ax3.set_title('Drift Scores Heatmap (Top 20)')\n plt.colorbar(im, ax=ax3)\n \n # 4. Distribution analysis\n ax4 = fig.add_subplot(gs[1, :2])\n if hasattr(self, 'distribution_metrics') and self.distribution_metrics is not None:\n transform_features = self.distribution_metrics[\n self.distribution_metrics['needs_transform']\n ].head(15)\n \n x = np.arange(len(transform_features))\n width = 0.35\n \n ax4.bar(x - width/2, transform_features['train_skew'], width, \n label='Train Skew', alpha=0.8, color='blue')\n ax4.bar(x + width/2, transform_features['test_skew'], width, \n label='Test Skew', alpha=0.8, color='orange')\n \n ax4.set_xlabel('Features')\n ax4.set_ylabel('Skewness')\n ax4.set_title('Features Requiring Transformation')\n ax4.set_xticks(x)\n ax4.set_xticklabels(transform_features.index, rotation=45, ha='right', fontsize=8)\n ax4.legend()\n ax4.axhline(y=0, color='k', linestyle='-', alpha=0.3)\n ax4.axhline(y=2, color='r', linestyle='--', alpha=0.3, label='Transform threshold')\n ax4.axhline(y=-2, color='r', linestyle='--', alpha=0.3)\n \n # 5. Feature categories pie chart\n ax5 = fig.add_subplot(gs[1, 2])\n categories = {\n 'Robust': len(self.intelligence_report['robust_features']),\n 'Stable': len(set(self.intelligence_report['stable_features']) - \n set(self.intelligence_report['robust_features'])),\n 'Unstable': len(self.intelligence_report['unstable_features']),\n 'To Remove': len(self.intelligence_report['features_to_remove']),\n 'To Regularize': len(self.intelligence_report['features_to_regularize']),\n 'To Transform': len(self.intelligence_report['features_to_transform'])\n }\n \n colors = ['darkgreen', 'green', 'orange', 'red', 'yellow', 'purple']\n ax5.pie(categories.values(), labels=categories.keys(), colors=colors, \n autopct='%1.1f%%', startangle=90)\n ax5.set_title('Feature Category Distribution')\n \n # 6. Autocorrelation patterns\n ax6 = fig.add_subplot(gs[2, 0])\n if self.feature_drift_scores is not None and len(self.feature_drift_scores) > 0:\n memory_features = self.feature_drift_scores[\n self.feature_drift_scores['has_memory']\n ].head(10)\n \n if len(memory_features) > 0:\n ax6.barh(range(len(memory_features)), \n memory_features['autocorr_1'], \n color=['red' if x < 0 else 'blue' for x in memory_features['autocorr_1']], \n alpha=0.7)\n ax6.set_yticks(range(len(memory_features)))\n ax6.set_yticklabels(memory_features.index, fontsize=8)\n ax6.set_xlabel('Lag-1 Autocorrelation')\n ax6.set_title('Features with Temporal Memory')\n ax6.axvline(x=0, color='k', linestyle='-', alpha=0.3)\n \n # 7. Intelligence summary\n ax7 = fig.add_subplot(gs[2:, 1:])\n ax7.axis('off')\n \n summary_text = \"ENHANCED TIME SERIES INTELLIGENCE SUMMARY\\n\" + \"=\"*50 + \"\\n\\n\"\n \n # Feature counts\n summary_text += \"FEATURE ANALYSIS RESULTS:\\n\"\n summary_text += f\" • Robust Features (consistent & stable): {len(self.intelligence_report['robust_features'])}\\n\"\n summary_text += f\" • Very Stable Features: {len(self.intelligence_report['very_stable_features'])}\\n\"\n summary_text += f\" • Improving Features: {len(self.intelligence_report['improving_features'])}\\n\"\n summary_text += f\" • Features to Remove: {len(self.intelligence_report['features_to_remove'])}\\n\"\n summary_text += f\" • Features to Regularize: {len(self.intelligence_report['features_to_regularize'])}\\n\"\n summary_text += f\" • Features to Transform: {len(self.intelligence_report['features_to_transform'])}\\n\"\n summary_text += f\" • Drifting Features: {len(self.intelligence_report['drifting_features'])}\\n\"\n summary_text += f\" • Seasonal Features: {len(self.intelligence_report['seasonal_features'])}\\n\\n\"\n \n summary_text += \"KEY INSIGHTS:\\n\"\n for i, insight in enumerate(self.intelligence_report['engineering_insights'][:6]):\n summary_text += f\"{i+1}. {insight}\\n\"\n \n summary_text += \"\\nTOP RECOMMENDATIONS:\\n\"\n for i, rec in enumerate(self.intelligence_report['model_recommendations'][:5]):\n summary_text += f\"{i+1}. {rec}\\n\"\n \n # Top robust features\n if self.intelligence_report['robust_features']:\n summary_text += f\"\\nTOP ROBUST FEATURES: {', '.join(self.intelligence_report['robust_features'][:10])}\"\n \n ax7.text(0.05, 0.95, summary_text, transform=ax7.transAxes, \n fontsize=10, verticalalignment='top', fontfamily='monospace',\n bbox=dict(boxstyle='round', facecolor='lightblue', alpha=0.8))\n \n plt.suptitle('Enhanced Time Series Intelligence Analysis', fontsize=18, y=0.98)\n plt.tight_layout()\n \n if CFG.save_figures:\n plt.savefig('enhanced_time_series_intelligence.png', dpi=CFG.figure_dpi, bbox_inches='tight')\n plt.show()\n \n return fig\n\n# Enhanced Intelligent XGBoost\nclass EnhancedIntelligentXGBoost:\n \"\"\"XGBoost model with improved intelligence integration\"\"\"\n \n def __init__(self, intelligence_report, base_params=None):\n self.intelligence = intelligence_report\n self.base_params = base_params or {\n 'n_estimators': 1500,\n 'max_depth': 8,\n 'learning_rate': 0.008,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'reg_alpha': 1.5,\n 'reg_lambda': 1.5,\n 'gamma': 0.1,\n 'min_child_weight': 3,\n 'random_state': 42,\n 'n_jobs': -1,\n 'verbosity': 0,\n 'objective': 'reg:squarederror',\n 'eval_metric': 'rmse',\n 'tree_method': 'hist'\n }\n self.models = [] # For ensemble\n self.scalers = []\n self.features_to_use = None\n self.feature_processors = {}\n self.transformation_pipeline = {}\n \n def prepare_features(self, X):\n \"\"\"Prepare features with intelligent transformations\"\"\"\n X_prepared = X.copy()\n \n # Apply recommended transformations\n transform_recommendations = self.intelligence.get('transformation_recommendations', {})\n \n for feat in X_prepared.columns:\n # Apply specific transformations\n if feat in transform_recommendations:\n transform = transform_recommendations[feat]\n if transform == 'log' and X_prepared[feat].min() >= 0:\n X_prepared[feat] = np.log1p(X_prepared[feat])\n self.transformation_pipeline[feat] = 'log'\n elif transform == 'sqrt_neg' and X_prepared[feat].min() >= 0:\n X_prepared[feat] = np.sqrt(X_prepared[feat])\n self.transformation_pipeline[feat] = 'sqrt'\n elif transform == 'rank':\n X_prepared[feat] = rankdata(X_prepared[feat]) / len(X_prepared)\n self.transformation_pipeline[feat] = 'rank'\n elif transform == 'robust_scale':\n median = X_prepared[feat].median()\n mad = (X_prepared[feat] - median).abs().median()\n X_prepared[feat] = (X_prepared[feat] - median) / (mad + 1e-8)\n self.transformation_pipeline[feat] = 'robust_scale'\n \n # Additional transformations based on feature type\n elif feat in self.intelligence.get('unstable_features', []):\n # Use percentile transformation for unstable features\n try:\n X_prepared[feat] = pd.qcut(X_prepared[feat].astype(np.float32), \n q=20, labels=False, duplicates='drop') / 20.0\n except:\n X_prepared[feat] = rankdata(X_prepared[feat]) / len(X_prepared)\n self.transformation_pipeline[feat] = 'percentile'\n \n elif feat in self.intelligence.get('drifting_features', []):\n # Clip extreme values for drifting features\n lower = X_prepared[feat].quantile(0.01)\n upper = X_prepared[feat].quantile(0.99)\n X_prepared[feat] = X_prepared[feat].clip(lower, upper)\n self.transformation_pipeline[feat] = 'clip'\n \n elif feat in self.intelligence.get('seasonal_features', []):\n # Detrend seasonal features\n x = np.arange(len(X_prepared))\n trend = np.polyfit(x, X_prepared[feat], 1)\n X_prepared[feat] = X_prepared[feat] - (trend[0] * x + trend[1])\n self.transformation_pipeline[feat] = 'detrend'\n \n return X_prepared\n \n def train_ensemble(self, X_train, y_train, X_val=None, y_val=None):\n \"\"\"Train ensemble with different configurations\"\"\"\n print(\"\\nTraining Enhanced Intelligent XGBoost Ensemble...\")\n \n # Remove harmful features\n features_to_remove = self.intelligence.get('features_to_remove', [])\n self.features_to_use = [f for f in X_train.columns if f not in features_to_remove]\n \n # Prioritize robust features\n robust_features = self.intelligence.get('robust_features', [])\n robust_in_use = [f for f in robust_features if f in self.features_to_use]\n \n print(f\" Using {len(self.features_to_use)} features ({len(robust_in_use)} robust)\")\n \n X_train_filtered = X_train[self.features_to_use]\n X_train_prepared = self.prepare_features(X_train_filtered)\n \n # Prepare validation data if provided\n if X_val is not None:\n X_val_filtered = X_val[self.features_to_use]\n X_val_prepared = self.prepare_features(X_val_filtered)\n \n # Define ensemble configurations\n ensemble_configs = [\n # Conservative model with high regularization\n {\n 'n_estimators': 1200,\n 'max_depth': 6,\n 'learning_rate': 0.005,\n 'reg_alpha': 3.0,\n 'reg_lambda': 3.0,\n 'subsample': 0.7,\n 'colsample_bytree': 0.7,\n 'min_child_weight': 5\n },\n # Balanced model\n {\n 'n_estimators': 1500,\n 'max_depth': 8,\n 'learning_rate': 0.008,\n 'reg_alpha': 2.0,\n 'reg_lambda': 2.0,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'min_child_weight': 3\n },\n # Less conservative model\n {\n 'n_estimators': 1000,\n 'max_depth': 10,\n 'learning_rate': 0.01,\n 'reg_alpha': 1.0,\n 'reg_lambda': 1.0,\n 'subsample': 0.85,\n 'colsample_bytree': 0.85,\n 'min_child_weight': 2\n }\n ]\n \n # Adjust configurations based on intelligence\n unstable_ratio = len([f for f in self.features_to_use \n if f in self.intelligence.get('unstable_features', [])]) / len(self.features_to_use)\n \n if unstable_ratio > 0.3:\n print(f\" High unstable ratio ({unstable_ratio:.1%}), increasing regularization\")\n for config in ensemble_configs:\n config['reg_alpha'] *= 1.5\n config['reg_lambda'] *= 1.5\n config['learning_rate'] *= 0.8\n \n # Train models\n for i, config in enumerate(ensemble_configs):\n print(f\"\\n Training model {i+1}/{len(ensemble_configs)}...\")\n \n # Create scaler\n scaler = RobustScaler()\n X_train_scaled = scaler.fit_transform(X_train_prepared)\n self.scalers.append(scaler)\n \n # Update config with base params\n model_params = self.base_params.copy()\n model_params.update(config)\n \n # Train model\n model = XGBRegressor(**model_params)\n \n if X_val is not None and y_val is not None:\n X_val_scaled = scaler.transform(X_val_prepared)\n model.fit(\n X_train_scaled, y_train,\n eval_set=[(X_val_scaled, y_val)],\n early_stopping_rounds=100,\n verbose=False\n )\n else:\n model.fit(X_train_scaled, y_train)\n \n self.models.append(model)\n \n # Print feature importance summary\n importances = pd.Series(model.feature_importances_, index=self.features_to_use)\n top_features = importances.nlargest(10)\n print(f\" Top features: {', '.join(top_features.index[:5])}\")\n \n print(f\"\\n Ensemble training complete with {len(self.models)} models\")\n print(f\" Applied transformations: {set(self.transformation_pipeline.values())}\")\n \n return self\n \n def predict(self, X):\n \"\"\"Make ensemble predictions\"\"\"\n X_filtered = X[self.features_to_use]\n X_prepared = self.prepare_features(X_filtered)\n \n predictions = []\n for model, scaler in zip(self.models, self.scalers):\n X_scaled = scaler.transform(X_prepared)\n pred = model.predict(X_scaled)\n predictions.append(pred)\n \n # Weighted average based on model conservativeness\n weights = [0.4, 0.4, 0.2] # More weight on conservative models\n ensemble_pred = np.average(predictions, axis=0, weights=weights)\n \n return ensemble_pred\n \n def get_feature_importance_report(self):\n \"\"\"Get aggregated feature importance\"\"\"\n if not self.models:\n return None\n \n # Aggregate importances across models\n importance_dict = {}\n for model in self.models:\n for feat, imp in zip(self.features_to_use, model.feature_importances_):\n if feat not in importance_dict:\n importance_dict[feat] = []\n importance_dict[feat].append(imp)\n \n # Calculate mean importance\n mean_importance = {feat: np.mean(imps) for feat, imps in importance_dict.items()}\n \n importance_df = pd.DataFrame({\n 'feature': list(mean_importance.keys()),\n 'importance': list(mean_importance.values())\n }).sort_values('importance', ascending=False)\n \n # Add feature categories\n importance_df['category'] = importance_df['feature'].apply(lambda x: \n 'robust' if x in self.intelligence.get('robust_features', []) else\n 'very_stable' if x in self.intelligence.get('very_stable_features', []) else\n 'stable' if x in self.intelligence.get('stable_features', []) else\n 'unstable' if x in self.intelligence.get('unstable_features', []) else\n 'other'\n )\n \n # Add transformation info\n importance_df['transformation'] = importance_df['feature'].apply(\n lambda x: self.transformation_pipeline.get(x, 'none')\n )\n \n return importance_df\n\n# Enhanced Feature Selector\nclass EnhancedIntelligentFeatureSelector:\n def __init__(self, intelligence_report):\n self.intelligence = intelligence_report\n self.selected_features = None\n self.feature_scores = None\n \n def select_features(self, X, y, n_features=None, min_features=100):\n \"\"\"Select features with enhanced intelligence\"\"\"\n print(f\"\\nSelecting features with enhanced intelligence...\")\n \n # Initialize feature scores\n feature_scores = {}\n \n # 1. XGBoost importance with better parameters\n xgb_model = XGBRegressor(\n n_estimators=300,\n max_depth=8,\n learning_rate=0.03,\n subsample=0.8,\n colsample_bytree=0.8,\n reg_alpha=1.5,\n reg_lambda=1.5,\n random_state=42,\n verbosity=0\n )\n xgb_model.fit(X, y)\n feature_scores['xgb'] = pd.Series(xgb_model.feature_importances_, index=X.columns)\n \n # 2. Random Forest importance for comparison\n rf_model = RandomForestRegressor(\n n_estimators=200,\n max_depth=10,\n min_samples_split=20,\n random_state=42,\n n_jobs=-1\n )\n rf_model.fit(X, y)\n feature_scores['rf'] = pd.Series(rf_model.feature_importances_, index=X.columns)\n \n # 3. Mutual information\n mi_scores = mutual_info_regression(X, y, random_state=42)\n feature_scores['mi'] = pd.Series(mi_scores, index=X.columns)\n \n # 4. F-statistics\n f_scores = f_regression(X, y)[0]\n feature_scores['f_stat'] = pd.Series(f_scores, index=X.columns)\n \n # 5. Stability scores from intelligence\n stability_scores = {}\n for feat in X.columns:\n if feat in self.intelligence.get('robust_features', []):\n stability_scores[feat] = 1.5 # Bonus for robust features\n elif feat in self.intelligence.get('very_stable_features', []):\n stability_scores[feat] = 1.2\n elif feat in self.intelligence.get('stable_features', []):\n stability_scores[feat] = 1.0\n elif feat in self.intelligence.get('improving_features', []):\n stability_scores[feat] = 0.9\n elif feat in self.intelligence.get('unstable_features', []):\n stability_scores[feat] = 0.3\n else:\n stability_scores[feat] = 0.7\n \n feature_scores['stability'] = pd.Series(stability_scores)\n \n # Normalize scores\n for key in feature_scores:\n scores = feature_scores[key]\n if scores.max() > scores.min():\n feature_scores[key] = (scores - scores.min()) / (scores.max() - scores.min())\n \n # Combine scores with adaptive weights\n if len(self.intelligence.get('robust_features', [])) > 20:\n # If we have many robust features, weight stability higher\n weights = {\n 'xgb': 0.25,\n 'rf': 0.15,\n 'mi': 0.15,\n 'f_stat': 0.05,\n 'stability': 0.40\n }\n else:\n # Otherwise, balance importance and stability\n weights = {\n 'xgb': 0.30,\n 'rf': 0.20,\n 'mi': 0.20,\n 'f_stat': 0.10,\n 'stability': 0.20\n }\n \n combined_scores = sum(feature_scores[key] * weights[key] for key in weights)\n \n # Apply penalties and boosts\n for feat in self.intelligence.get('features_to_remove', []):\n if feat in combined_scores.index:\n combined_scores[feat] *= 0.1\n \n for feat in self.intelligence.get('robust_features', []):\n if feat in combined_scores.index:\n combined_scores[feat] *= 1.3\n \n for feat in self.intelligence.get('features_to_regularize', []):\n if feat in combined_scores.index:\n combined_scores[feat] *= 0.8\n \n # Store scores\n self.feature_scores = pd.DataFrame(feature_scores)\n self.feature_scores['combined'] = combined_scores\n self.feature_scores = self.feature_scores.sort_values('combined', ascending=False)\n \n # Select features\n if n_features is None:\n # Adaptive selection based on feature quality\n quality_threshold = self.feature_scores['combined'].quantile(0.4)\n n_features = max(\n min_features,\n len(self.feature_scores[self.feature_scores['combined'] > quality_threshold])\n )\n \n self.selected_features = self.feature_scores.head(n_features).index.tolist()\n \n # Ensure critical features are included\n critical_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'volume',\n 'liquidity_imbalance', 'amihud_illiquidity', 'bid_ask_spread',\n 'buying_pressure', 'net_pressure', 'flow_toxicity']\n \n for feat in critical_features:\n if (feat in X.columns and \n feat not in self.selected_features and\n feat not in self.intelligence.get('features_to_remove', [])):\n self.selected_features.append(feat)\n \n # Add top robust features if missing\n for feat in self.intelligence.get('robust_features', [])[:10]:\n if feat in X.columns and feat not in self.selected_features:\n self.selected_features.append(feat)\n \n print(f\"Selected {len(self.selected_features)} features\")\n print(f\" Including {len([f for f in self.selected_features if f in self.intelligence.get('robust_features', [])])} robust features\")\n print(f\" Including {len([f for f in self.selected_features if f in critical_features])} critical market features\")\n \n return self.selected_features\n\n# Main Pipeline\ndef main():\n print_section(\"LOADING DATA\")\n \n # Load data\n train_df = pd.read_parquet(CFG.train_path)\n test_df = pd.read_parquet(CFG.test_path)\n submission = pd.read_csv(CFG.sample_sub_path)\n \n print(f\"Train shape: {train_df.shape}\")\n print(f\"Test shape: {test_df.shape}\")\n \n # Optimize memory\n train_df = reduce_mem_usage(train_df, \"train\")\n test_df = reduce_mem_usage(test_df, \"test\")\n \n # Extract labels\n y_train = train_df['label']\n \n print_section(\"ENHANCED FEATURE ENGINEERING\")\n \n # Add market features\n train_df = add_market_features(train_df)\n test_df = add_market_features(test_df)\n \n # Add robust features\n train_df = create_robust_features(train_df, CFG.n_robust_features)\n test_df = create_robust_features(test_df, CFG.n_robust_features)\n \n # Add temporal-aware features\n train_df = create_temporal_aware_features(train_df, CFG.n_temporal_features)\n test_df = create_temporal_aware_features(test_df, CFG.n_temporal_features)\n \n # Add proprietary features\n train_df = create_proprietary_features(train_df, CFG.n_proprietary_features)\n test_df = create_proprietary_features(test_df, CFG.n_proprietary_features)\n \n # Get feature columns\n feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp']]\n \n # Create interaction features\n interaction_train = create_interaction_features(train_df, feature_cols, CFG.n_interaction_features)\n interaction_test = create_interaction_features(test_df, feature_cols, CFG.n_interaction_features)\n \n # Add interaction features\n for col in interaction_train.columns:\n train_df[col] = interaction_train[col]\n test_df[col] = interaction_test[col]\n \n # Update feature columns\n feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp']]\n \n print(f\"\\nTotal features created: {len(feature_cols)}\")\n \n print_section(\"BASELINE MODEL (WITHOUT INTELLIGENCE)\")\n \n # Baseline feature selection\n X_train = train_df[feature_cols]\n X_test = test_df[feature_cols]\n \n # Simple feature selection\n selector = SelectKBest(f_regression, k=min(200, len(feature_cols)))\n selector.fit(X_train, y_train)\n baseline_features = X_train.columns[selector.get_support()].tolist()\n \n X_train_baseline = X_train[baseline_features]\n X_test_baseline = X_test[baseline_features]\n \n # Train baseline model with better parameters\n baseline_model = XGBRegressor(\n n_estimators=1000,\n max_depth=8,\n learning_rate=0.01,\n subsample=0.8,\n colsample_bytree=0.8,\n reg_alpha=1.5,\n reg_lambda=1.5,\n random_state=42,\n verbosity=0\n )\n baseline_model.fit(X_train_baseline, y_train)\n baseline_pred = baseline_model.predict(X_test_baseline)\n \n # Baseline CV score\n baseline_cv_scores = []\n kf = KFold(n_splits=5, shuffle=True, random_state=42)\n for train_idx, val_idx in kf.split(X_train_baseline):\n X_cv_train = X_train_baseline.iloc[train_idx]\n y_cv_train = y_train.iloc[train_idx]\n X_cv_val = X_train_baseline.iloc[val_idx]\n y_cv_val = y_train.iloc[val_idx]\n \n cv_model = XGBRegressor(\n n_estimators=300,\n max_depth=8,\n learning_rate=0.03,\n subsample=0.8,\n colsample_bytree=0.8,\n random_state=42,\n verbosity=0\n )\n cv_model.fit(X_cv_train, y_cv_train)\n cv_pred = cv_model.predict(X_cv_val)\n cv_score = pearsonr(y_cv_val, cv_pred)[0]\n baseline_cv_scores.append(cv_score)\n \n baseline_cv_mean = np.mean(baseline_cv_scores)\n print(f\"\\nBaseline Model Performance:\")\n print(f\" Features: {len(baseline_features)}\")\n print(f\" CV Score: {baseline_cv_mean:.4f} (+/- {np.std(baseline_cv_scores):.4f})\")\n \n print_section(\"ENHANCED TIME SERIES INTELLIGENCE EXTRACTION\")\n \n # Check if timestamp reconstruction is available\n use_ts_insights = os.path.exists(CFG.timestamp_recon_path)\n \n if use_ts_insights:\n print(\"\\nTime series reconstruction found. Extracting enhanced intelligence...\")\n \n # Load and process timestamps\n try:\n t = pd.read_csv(CFG.timestamp_recon_path)['0'].to_numpy()\n t = pd.Series(t)\n t -= 10080\n t[t < 0] = 538149\n t = t.sort_values()\n t[t <= len(t)] = np.arange(t[t <= len(t)].shape[0])\n t = t.sort_index()\n \n # Create mapping\n mapping = pd.Series(np.arange(len(test_df)), index=t.to_numpy()).sort_index()\n \n # Create ordered test set\n test_df_ordered = test_df.iloc[mapping.values]\n \n # Extract enhanced intelligence\n intelligence_extractor = EnhancedTimeSeriesIntelligenceExtractor()\n intelligence_report = intelligence_extractor.analyze_with_time_series(\n train_df, test_df_ordered, y_train, baseline_features\n )\n \n # Visualize intelligence\n intelligence_extractor.visualize_enhanced_intelligence()\n \n print_section(\"ENHANCED INTELLIGENT MODEL\")\n \n # Enhanced feature selection\n intelligent_selector = EnhancedIntelligentFeatureSelector(intelligence_report)\n selected_features = intelligent_selector.select_features(\n X_train, y_train, n_features=None, min_features=150\n )\n \n X_train_selected = X_train[selected_features]\n X_test_selected = X_test[selected_features]\n \n # Train enhanced intelligent model\n intelligent_model = EnhancedIntelligentXGBoost(intelligence_report)\n \n # Use 20% of training data for validation\n X_train_split, X_val_split, y_train_split, y_val_split = train_test_split(\n X_train_selected, y_train, test_size=0.2, random_state=42\n )\n \n intelligent_model.train_ensemble(\n X_train_split, y_train_split,\n X_val_split, y_val_split\n )\n \n # Make predictions\n intelligent_pred = intelligent_model.predict(X_test_selected)\n \n # Cross-validation for intelligent model\n intelligent_cv_scores = []\n for train_idx, val_idx in kf.split(X_train_selected):\n X_cv_train = X_train_selected.iloc[train_idx]\n y_cv_train = y_train.iloc[train_idx]\n X_cv_val = X_train_selected.iloc[val_idx]\n y_cv_val = y_train.iloc[val_idx]\n \n cv_model = EnhancedIntelligentXGBoost(intelligence_report)\n cv_model.train_ensemble(X_cv_train, y_cv_train)\n cv_pred = cv_model.predict(X_cv_val)\n cv_score = pearsonr(y_cv_val, cv_pred)[0]\n intelligent_cv_scores.append(cv_score)\n \n intelligent_cv_mean = np.mean(intelligent_cv_scores)\n print(f\"\\nEnhanced Intelligent Model Performance:\")\n print(f\" Features: {len(selected_features)}\")\n print(f\" CV Score: {intelligent_cv_mean:.4f} (+/- {np.std(intelligent_cv_scores):.4f})\")\n print(f\" Improvement: {(intelligent_cv_mean - baseline_cv_mean) / baseline_cv_mean * 100:.2f}%\")\n \n # Display intelligence summary\n print(\"\\nIntelligence Summary:\")\n print(f\" - Robust features identified: {len(intelligence_report.get('robust_features', []))}\")\n print(f\" - Features removed: {len(intelligence_report.get('features_to_remove', []))}\")\n print(f\" - Features regularized: {len(intelligence_report.get('features_to_regularize', []))}\")\n print(f\" - Features transformed: {len(intelligence_report.get('features_to_transform', []))}\")\n print(f\" - Improving features: {len(intelligence_report.get('improving_features', []))}\")\n \n # Get feature importance\n importance_report = intelligent_model.get_feature_importance_report()\n if importance_report is not None:\n print(\"\\nTop 20 Features in Enhanced Model:\")\n print(importance_report.head(20).to_string(index=False))\n \n except Exception as e:\n print(f\"\\nError processing time series data: {e}\")\n print(\"Falling back to baseline approach.\")\n import traceback\n traceback.print_exc()\n intelligent_pred = baseline_pred\n intelligence_report = None\n else:\n print(\"\\nTime series reconstruction not available.\")\n intelligent_pred = baseline_pred\n intelligence_report = None\n \n print_section(\"CREATING SUBMISSIONS\")\n \n # Create submissions\n submission_baseline = submission.copy()\n submission_baseline['prediction'] = baseline_pred\n submission_baseline.to_csv('submission_baseline.csv', index=False)\n print(\"Created: submission_baseline.csv\")\n \n if intelligence_report:\n submission_intelligent = submission.copy()\n submission_intelligent['prediction'] = intelligent_pred\n submission_intelligent.to_csv('submission_enhanced.csv', index=False)\n print(\"Created: submission_enhanced.csv\")\n \n # Create weighted ensemble\n if intelligent_cv_mean > baseline_cv_mean:\n # Calculate adaptive weights based on CV performance\n total = intelligent_cv_mean + baseline_cv_mean\n intel_weight = intelligent_cv_mean / total\n base_weight = baseline_cv_mean / total\n \n # Boost intelligent weight if significantly better\n if intelligent_cv_mean > baseline_cv_mean * 1.02:\n intel_weight = min(0.8, intel_weight * 1.2)\n base_weight = 1 - intel_weight\n else:\n intel_weight = 0.4\n base_weight = 0.6\n \n ensemble_pred = intel_weight * intelligent_pred + base_weight * baseline_pred\n submission_ensemble = submission.copy()\n submission_ensemble['prediction'] = ensemble_pred\n submission_ensemble.to_csv('submission_ensemble.csv', index=False)\n print(f\"Created: submission_ensemble.csv (intelligent weight: {intel_weight:.1%})\")\n \n print_section(\"FINAL RECOMMENDATIONS\")\n \n print(\"\\n1. FURTHER IMPROVEMENTS:\")\n print(\" - Implement adversarial validation for feature selection\")\n print(\" - Use SHAP values to understand feature interactions\")\n print(\" - Try CatBoost/LightGBM for comparison\")\n print(\" - Implement pseudo-labeling with confidence thresholds\")\n print(\" - Use test-time augmentation for predictions\")\n \n print(\"\\n2. ENSEMBLE STRATEGIES:\")\n print(\" - Blend multiple time windows (recent vs all data)\")\n print(\" - Stack with neural networks for non-linear patterns\")\n print(\" - Use different feature subsets for diversity\")\n print(\" - Implement dynamic weighting based on market regime\")\n \n print(\"\\n3. PRODUCTION DEPLOYMENT:\")\n print(\" - Monitor feature drift in real-time\")\n print(\" - Implement online learning capabilities\")\n print(\" - Create feature importance dashboards\")\n print(\" - Set up A/B testing framework\")\n \n print_section(\"PIPELINE COMPLETED\")\n \n # Save enhanced reports\n if intelligence_report:\n # Save detailed report\n report_summary = {\n 'model_performance': {\n 'baseline_cv': baseline_cv_mean,\n 'intelligent_cv': intelligent_cv_mean,\n 'improvement': (intelligent_cv_mean - baseline_cv_mean) / baseline_cv_mean * 100\n },\n 'feature_counts': {\n 'total_created': len(feature_cols),\n 'baseline_selected': len(baseline_features),\n 'intelligent_selected': len(selected_features),\n 'robust_features': len(intelligence_report.get('robust_features', [])),\n 'removed_features': len(intelligence_report.get('features_to_remove', []))\n },\n 'intelligence_summary': {\n key: len(value) if isinstance(value, list) else value\n for key, value in intelligence_report.items()\n if not key.endswith('_recommendations')\n }\n }\n \n with open('enhanced_intelligence_report.json', 'w') as f:\n json.dump(report_summary, f, indent=2)\n print(\"\\nSaved: enhanced_intelligence_report.json\")\n \n # Save feature importance\n if 'importance_report' in locals() and importance_report is not None:\n importance_report.to_csv('enhanced_feature_importance.csv', index=False)\n print(\"Saved: enhanced_feature_importance.csv\")\n \n print(\"\\n\" + \"=\"*60)\n print(\"CONCLUSION:\")\n print(\"=\"*60)\n print(\"The enhanced pipeline demonstrates how to:\")\n print(\"1. Generate better pseudo-labels using ensemble methods\")\n print(\"2. Analyze stability across multiple time windows\")\n print(\"3. Apply intelligent feature-specific transformations\")\n print(\"4. Balance stability and predictive power in feature selection\")\n print(\"5. Use ensemble models with adaptive regularization\")\n print(\"6. Create production-ready intelligence reports\")\n print(\"=\"*60)\n \n return {\n 'baseline_cv': baseline_cv_mean,\n 'intelligent_cv': intelligent_cv_mean if intelligence_report else None,\n 'intelligence_report': intelligence_report,\n 'feature_cols': feature_cols\n }\n\nif __name__ == \"__main__\":\n results = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}