{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":12072741,"sourceType":"datasetVersion","datasetId":7599534}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pipeline for regression tasks using XGBoost and LightGBM with ensemble modeling, time-based weighting, and flexible data loading capabilities.\n\n### Core Classes\n\n1. **`DatasetConfig`**: Configuration for individual dataset files\n2. **`ModelConfig`**: Main configuration class for the entire pipeline\n3. **`WeightCalculator`**: Time-based sample weight generation\n4. **`DataProcessor`**: Data loading, merging, and preprocessing\n5. **`ModelTrainer`**: Model training with different algorithms and data subsets\n6. **`EnsembleManager`**: Ensemble creation and evaluation\n7. **`XGBoostLightGBMPipeline`**: Main orchestrator class\n\n### Key Features\n\n- **Flexible Dataset Configuration**: Support for multiple files with automatic merging\n- **Feature Selection**: Specify exact features to keep from each file\n- **Time-Based Weights**: Exponential decay to emphasize recent data\n- **Multiple Model Variants**: Train models on different data subsets\n- **Ensemble Strategies**: Simple average and performance-weighted ensembles","metadata":{"execution":{"iopub.status.busy":"2025-06-10T11:01:08.854725Z","iopub.execute_input":"2025-06-10T11:01:08.854979Z","iopub.status.idle":"2025-06-10T11:01:08.864954Z","shell.execute_reply.started":"2025-06-10T11:01:08.854960Z","shell.execute_reply":"2025-06-10T11:01:08.863892Z"}}},{"cell_type":"markdown","source":"# Regimes Using HMM (Basic Version with 2 Regimes Only) : https://www.kaggle.com/code/digantabhattacharya/drw-regime-detection-using-hmm-step-1 ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\nfrom xgboost import XGBRegressor\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import KFold\nfrom scipy.stats import pearsonr\nimport time\nimport gc\nimport warnings\nimport psutil\nfrom datetime import datetime, timedelta\nfrom typing import Dict, Tuple, List, Optional, Any\nfrom dataclasses import dataclass, field\nimport logging\nimport os\nfrom joblib import Parallel, delayed\nimport multiprocessing\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n\n# Configure warnings and logging\nwarnings.filterwarnings('ignore')\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\n\ndef print_header(title: str, width: int = 80):\n \"\"\"Print a formatted header for better visibility.\"\"\"\n print(f\"\\n{'='*width}\")\n print(f\"๐Ÿš€ {title}\")\n print(f\"{'='*width}\")\n\ndef print_step(step: str, substep: str = None):\n \"\"\"Print current step with clear formatting.\"\"\"\n if substep:\n print(f\"\\n โ””โ”€โ”€ {substep}\")\n else:\n print(f\"\\n๐Ÿ“‹ {step}\")\n\ndef print_progress(current: int, total: int, item_name: str = \"items\", eta: str = None):\n \"\"\"Print progress with percentage and ETA.\"\"\"\n percentage = (current / total) * 100 if total > 0 else 0\n eta_str = f\" | ETA: {eta}\" if eta else \"\"\n print(f\" ๐Ÿ”„ Progress: {current}/{total} {item_name} ({percentage:.1f}%){eta_str}\")\n\ndef print_result(metric_name: str, value: float, format_str: str = \".6f\"):\n \"\"\"Print a result metric with proper formatting.\"\"\"\n print(f\" โœ… {metric_name}: {value:{format_str}}\")\n\ndef print_error(error_msg: str):\n \"\"\"Print error message with clear formatting.\"\"\"\n print(f\" โŒ ERROR: {error_msg}\")\n\ndef print_warning(warning_msg: str):\n \"\"\"Print warning message with clear formatting.\"\"\"\n print(f\" โš ๏ธ WARNING: {warning_msg}\")\n\ndef get_memory_usage() -> float:\n \"\"\"Get current memory usage in GB.\"\"\"\n try:\n process = psutil.Process()\n return process.memory_info().rss / (1024**3) # Convert to GB\n except:\n return 0.0\n\ndef print_memory_usage(stage: str):\n \"\"\"Print current memory usage for a specific stage.\"\"\"\n memory_gb = get_memory_usage()\n print(f\" ๐Ÿง  Memory usage at {stage}: {memory_gb:.2f} GB\")\n\ndef cleanup_memory(variables_to_delete: List[str] = None, local_vars: dict = None):\n \"\"\"\n Comprehensive memory cleanup function.\n \n Args:\n variables_to_delete: List of variable names to delete from local scope\n local_vars: Local variables dictionary (usually locals())\n \"\"\"\n print_step(\"Memory cleanup before model training\")\n \n # Get initial memory usage\n initial_memory = get_memory_usage()\n print_memory_usage(\"cleanup start\")\n \n # Delete specified variables if provided\n if variables_to_delete and local_vars:\n for var_name in variables_to_delete:\n if var_name in local_vars:\n print(f\" ๐Ÿ—‘๏ธ Deleting variable: {var_name}\")\n del local_vars[var_name]\n \n # Force garbage collection multiple times for thorough cleanup\n print(\" ๐Ÿงน Running garbage collection...\")\n collected_objects = 0\n for i in range(3): # Multiple passes for thorough cleanup\n collected = gc.collect()\n collected_objects += collected\n if collected > 0:\n print(f\" โœ… GC pass {i+1}: collected {collected} objects\")\n \n print(f\" ๐Ÿ“Š Total objects collected: {collected_objects}\")\n \n # Clear any remaining unreferenced objects\n gc.collect()\n \n # Get final memory usage\n final_memory = get_memory_usage()\n memory_freed = initial_memory - final_memory\n \n print_memory_usage(\"cleanup complete\")\n if memory_freed > 0:\n print(f\" โœ… Memory freed: {memory_freed:.2f} GB\")\n else:\n print(f\" โ„น๏ธ Memory usage stable (difference: {abs(memory_freed):.2f} GB)\")\n \n logger.info(f\"Memory cleanup completed: {initial_memory:.2f}GB โ†’ {final_memory:.2f}GB\")\n\ndef analyze_dataframe_columns(df: pd.DataFrame, dataset_name: str = \"DataFrame\"):\n \"\"\"\n Analyze and report on DataFrame columns for debugging purposes.\n \n Args:\n df: DataFrame to analyze\n dataset_name: Name for reporting\n \"\"\"\n print(f\"\\n๐Ÿ” Column Analysis for {dataset_name}:\")\n print(f\" ๐Ÿ“Š Shape: {df.shape}\")\n print(f\" ๐Ÿ“‹ Total columns: {len(df.columns)}\")\n \n # Group columns by type\n numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()\n object_cols = df.select_dtypes(include=['object']).columns.tolist()\n datetime_cols = df.select_dtypes(include=['datetime64']).columns.tolist()\n \n print(f\" ๐Ÿ”ข Numeric columns: {len(numeric_cols)}\")\n print(f\" ๐Ÿ“ Object columns: {len(object_cols)}\")\n print(f\" ๐Ÿ“… Datetime columns: {len(datetime_cols)}\")\n \n if datetime_cols:\n print(f\" ๐Ÿ“… Datetime: {datetime_cols}\")\n \n # Check for potential ID/timestamp columns\n potential_id_cols = []\n for col in df.columns:\n col_lower = col.lower()\n if any(pattern in col_lower for pattern in ['id', 'timestamp', 'time', 'date', 'datetime', 'ts']):\n potential_id_cols.append(col)\n \n if potential_id_cols:\n print(f\" ๐Ÿ†” Potential ID/timestamp columns: {potential_id_cols}\")\n \n # Show first 20 columns\n if len(df.columns) > 20:\n print(f\" ๐Ÿ“‹ First 20 columns: {list(df.columns[:20])}\")\n print(f\" ๐Ÿ“‹ ... and {len(df.columns) - 20} more\")\n else:\n print(f\" ๐Ÿ“‹ All columns: {list(df.columns)}\")\n \n # Check index\n if hasattr(df.index, 'name') and df.index.name:\n print(f\" ๐Ÿ“… Index: {df.index.name} (type: {df.index.dtype})\")\n \n return {\n 'numeric_cols': numeric_cols,\n 'object_cols': object_cols, \n 'datetime_cols': datetime_cols,\n 'potential_id_cols': potential_id_cols\n }\n\n@dataclass\nclass DatasetConfig:\n \"\"\"Configuration for individual dataset files (supports CSV and Parquet formats).\"\"\"\n file_path: str # Path to CSV or Parquet file\n feature_columns: List[str] # Specific columns to keep from this file\n id_columns: List[str] # ID/merge columns (timestamp, ID, etc.)\n dataset_name: str # Human readable name for logging\n is_required: bool = True # Whether this dataset is required for pipeline to continue\n\n@dataclass\nclass ModelConfig:\n \"\"\"Configuration class for model parameters and settings.\"\"\"\n \n # Dataset configurations\n TRAIN_DATASETS: List[DatasetConfig] = field(default_factory=list)\n TEST_DATASETS: List[DatasetConfig] = field(default_factory=list)\n \n # Column names\n TARGET_COLUMN: str = \"label\"\n ID_COLUMN: str = \"ID\"\n TIMESTAMP_COLUMN: str = \"timestamp\"\n \n # Cross-validation settings\n N_FOLDS: int = 5\n RANDOM_STATE: int = 42\n \n # Time decay settings\n DECAY_FACTOR: float = 0.95\n \n # Model parameters\n XGB_PARAMS: Dict[str, Any] = field(default_factory=dict)\n LGBM_PARAMS: Dict[str, Any] = field(default_factory=dict)\n \n # Feature selection\n SELECTED_FEATURES: List[str] = field(default_factory=list)\n \n # Model ensemble configurations\n MODEL_CONFIGS: List[Dict[str, Any]] = field(default_factory=lambda: [\n {\"name\": \"Full Dataset (100%)\", \"percent\": 1.00, \"priority\": 1},\n {\"name\": \"Recent Data (75%)\", \"percent\": 0.75, \"priority\": 2},\n {\"name\": \"Recent Data (50%)\", \"percent\": 0.50, \"priority\": 3}\n ])\n \n # Output settings\n SUBMISSION_FILENAME: str = \"submission_ensemble_XGB_LGB.csv\"\n RESULTS_FILENAME: str = \"ensemble_results.csv\"\n \n # Performance optimization settings\n REDUCE_MEMORY_USAGE: bool = True # Enable memory usage reduction\n ADD_ENGINEERED_FEATURES: bool = False # Enable feature engineering\n \n # Multi-GPU settings\n USE_MULTI_GPU: bool = True # Enable multi-GPU training\n GPU_DEVICES: List[int] = field(default_factory=lambda: [0, 1]) # GPU device IDs\n PARALLEL_FOLD_TRAINING: bool = True # Train folds in parallel across GPUs\n \n # Ensemble weight configurations\n CUSTOM_ENSEMBLE_WEIGHTS: List[float] = field(default_factory=list) # Custom weights for final ensemble [XGB_weight, LGB_weight]\n INDIVIDUAL_MODEL_WEIGHTS: Dict[str, float] = field(default_factory=dict) # Custom weights for individual models\n ENSEMBLE_STRATEGY: str = \"learner_level\" # \"learner_level\", \"individual_models\", or \"performance_based\"\n \n def validate_weights(self) -> Tuple[bool, str]:\n \"\"\"\n Validate ensemble weight configurations.\n \n Returns:\n Tuple of (is_valid, error_message)\n \"\"\"\n if self.ENSEMBLE_STRATEGY == \"individual_models\":\n if not self.INDIVIDUAL_MODEL_WEIGHTS:\n return False, \"INDIVIDUAL_MODEL_WEIGHTS must be specified when ENSEMBLE_STRATEGY is 'individual_models'\"\n \n # Expected model names based on MODEL_CONFIGS\n expected_models = []\n for algorithm in ['XGB', 'LGB']:\n for model_config in self.MODEL_CONFIGS:\n model_name = f\"{algorithm}_{model_config['name'].replace(' ', '_').replace('(', '').replace(')', '')}\"\n expected_models.append(model_name)\n \n # Check if all expected models have weights\n missing_weights = [model for model in expected_models if model not in self.INDIVIDUAL_MODEL_WEIGHTS]\n if missing_weights:\n return False, f\"Missing weights for models: {missing_weights}\"\n \n # Check for extra weights\n extra_weights = [model for model in self.INDIVIDUAL_MODEL_WEIGHTS if model not in expected_models]\n if extra_weights:\n return False, f\"Unexpected model weights specified: {extra_weights}. Expected models: {expected_models}\"\n \n # Check that weights are positive\n negative_weights = [model for model, weight in self.INDIVIDUAL_MODEL_WEIGHTS.items() if weight < 0]\n if negative_weights:\n return False, f\"Negative weights not allowed for models: {negative_weights}\"\n \n elif self.ENSEMBLE_STRATEGY == \"learner_level\":\n if self.CUSTOM_ENSEMBLE_WEIGHTS:\n if len(self.CUSTOM_ENSEMBLE_WEIGHTS) != 2:\n return False, \"CUSTOM_ENSEMBLE_WEIGHTS must contain exactly 2 weights [XGB_weight, LGB_weight]\"\n \n if any(w < 0 for w in self.CUSTOM_ENSEMBLE_WEIGHTS):\n return False, \"Negative weights not allowed in CUSTOM_ENSEMBLE_WEIGHTS\"\n \n elif self.ENSEMBLE_STRATEGY == \"performance_based\":\n # Performance-based ensemble doesn't require any additional configuration\n # Weights are automatically calculated based on CV scores\n pass\n \n else:\n return False, f\"Invalid ENSEMBLE_STRATEGY: {self.ENSEMBLE_STRATEGY}. Must be 'learner_level', 'individual_models', or 'performance_based'\"\n \n return True, \"\"\n\nclass WeightCalculator:\n \"\"\"Calculate time-based sample weights with exponential decay.\"\"\"\n \n @staticmethod\n def create_time_weights(n_samples: int, decay_factor: float = 0.95) -> np.ndarray:\n \"\"\"\n Create exponentially decaying weights based on sample position.\n More recent samples (higher indices) get higher weights.\n \n Args:\n n_samples: Number of samples\n decay_factor: Controls the rate of decay (0.95 = 5% decay per time unit)\n \n Returns:\n Array of sample weights normalized to sum to n_samples\n \"\"\"\n if n_samples == 0:\n return np.array([])\n \n print(f\" โš–๏ธ Creating time weights for {n_samples:,} samples (decay: {decay_factor})\")\n \n positions = np.arange(n_samples)\n # Normalize positions to [0, 1] range\n normalized_positions = positions / max(1, n_samples - 1)\n # Apply exponential weighting\n weights = decay_factor ** (1 - normalized_positions)\n # Normalize weights to sum to n_samples (maintains scale)\n weights = weights * n_samples / weights.sum()\n \n print(f\" โœ… Weight range: [{weights.min():.4f}, {weights.max():.4f}], mean: {weights.mean():.4f}\")\n \n return weights\n\nclass DataProcessor:\n \"\"\"Handle data loading, merging, and preprocessing with flexible dataset configurations.\"\"\"\n \n def __init__(self, config: ModelConfig):\n self.config = config\n print_step(\"Data Processor Initialized\")\n print(f\" ๐Ÿ“Š Train datasets configured: {len(self.config.TRAIN_DATASETS)}\")\n print(f\" ๐Ÿ“Š Test datasets configured: {len(self.config.TEST_DATASETS)}\")\n print(f\" ๐ŸŽฏ Selected features: {len(self.config.SELECTED_FEATURES)}\")\n print(f\" ๐Ÿง  Memory optimization: {self.config.REDUCE_MEMORY_USAGE}\")\n print(f\" โš™๏ธ Feature engineering: {self.config.ADD_ENGINEERED_FEATURES}\")\n logger.info(f\"DataProcessor initialized with {len(self.config.TRAIN_DATASETS)} train and {len(self.config.TEST_DATASETS)} test datasets\")\n \n def reduce_mem_usage(self, dataframe: pd.DataFrame, dataset_name: str) -> pd.DataFrame:\n \"\"\"\n Reduce memory usage of dataframe by optimizing ONLY numeric data types.\n COMPLETELY preserves object/string columns (including string timestamps) and datetime columns.\n \n Args:\n dataframe: DataFrame to optimize\n dataset_name: Name for logging purposes\n \n Returns:\n Memory-optimized dataframe with preserved non-numeric columns\n \"\"\"\n if not self.config.REDUCE_MEMORY_USAGE:\n return dataframe\n \n print(f\" ๐Ÿง  Reducing memory usage for: {dataset_name}\")\n initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n \n # Identify columns to preserve (anything non-numeric)\n preserved_columns = set()\n numeric_columns = []\n \n # Add configured ID and timestamp columns to preserved\n if self.config.ID_COLUMN:\n preserved_columns.add(self.config.ID_COLUMN)\n if self.config.TIMESTAMP_COLUMN:\n preserved_columns.add(self.config.TIMESTAMP_COLUMN)\n if self.config.TARGET_COLUMN:\n preserved_columns.add(self.config.TARGET_COLUMN)\n \n # Categorize all columns by type\n for col in dataframe.columns:\n col_type = dataframe[col].dtype\n \n # Preserve object/string columns (including string timestamps)\n if col_type == 'object':\n preserved_columns.add(col)\n continue\n \n # Preserve datetime columns\n if pd.api.types.is_datetime64_any_dtype(dataframe[col]):\n preserved_columns.add(col)\n continue\n \n # Preserve category and bool columns\n if col_type in ['category', 'bool']:\n preserved_columns.add(col)\n continue\n \n # Only optimize numeric columns (int and float)\n if str(col_type)[:3] in ['int', 'flo']:\n numeric_columns.append(col)\n \n print(f\" ๐Ÿ›ก๏ธ Preserved columns: {len(preserved_columns)} (object/datetime/category/bool)\")\n print(f\" ๐Ÿ”ข Numeric columns to optimize: {len(numeric_columns)}\")\n \n if preserved_columns:\n # Show types of preserved columns for debugging\n preserved_types = {}\n for col in list(preserved_columns)[:5]: # Show first 5 for brevity\n if col in dataframe.columns:\n preserved_types[col] = str(dataframe[col].dtype)\n if preserved_types:\n print(f\" ๐Ÿ“ Sample preserved column types: {preserved_types}\")\n \n optimized_count = 0\n for col in numeric_columns:\n col_type = dataframe[col].dtype\n \n try:\n c_min = dataframe[col].min()\n c_max = dataframe[col].max()\n \n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n dataframe[col] = dataframe[col].astype(np.int8)\n optimized_count += 1\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n dataframe[col] = dataframe[col].astype(np.int16)\n optimized_count += 1\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n dataframe[col] = dataframe[col].astype(np.int32)\n optimized_count += 1\n elif str(col_type)[:3] == 'flo':\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n dataframe[col] = dataframe[col].astype(np.float16)\n optimized_count += 1\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n dataframe[col] = dataframe[col].astype(np.float32)\n optimized_count += 1\n except Exception as e:\n print(f\" โš ๏ธ Could not optimize numeric column {col}: {str(e)}\")\n continue\n\n final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n reduction_pct = 100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage\n \n print(f\" ๐Ÿ“ˆ Memory usage before: {initial_mem_usage:.2f} MB\")\n print(f\" ๐Ÿ“‰ Memory usage after: {final_mem_usage:.2f} MB\")\n print(f\" โœ… Optimized {optimized_count}/{len(numeric_columns)} numeric columns\")\n print(f\" ๐Ÿ›ก๏ธ Preserved {len(preserved_columns)} non-numeric columns unchanged\")\n print(f\" ๐Ÿ“Š Overall memory reduction: {reduction_pct:.1f}%\")\n \n logger.info(f\"Memory optimization for {dataset_name}: {initial_mem_usage:.2f}MB โ†’ {final_mem_usage:.2f}MB ({reduction_pct:.1f}% reduction), preserved {len(preserved_columns)} columns\")\n \n return dataframe\n \n def add_features(self, df: pd.DataFrame, dataset_name: str) -> pd.DataFrame:\n \"\"\"\n Add engineered features based on market microstructure data.\n \n Args:\n df: Input dataframe with basic market data\n dataset_name: Name for logging purposes\n \n Returns:\n Dataframe with additional engineered features\n \"\"\"\n if not self.config.ADD_ENGINEERED_FEATURES:\n return df\n \n print(f\" โš™๏ธ Engineering features for: {dataset_name}\")\n \n # Check if required columns exist\n required_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n missing_cols = [col for col in required_cols if col not in df.columns]\n \n if missing_cols:\n print_warning(f\"Missing columns for feature engineering: {missing_cols}. Skipping feature engineering.\")\n return df\n \n data = df.copy()\n initial_features = len(data.columns)\n \n # Market microstructure features\n data['bid_ask_spread_proxy'] = data['ask_qty'] - data['bid_qty']\n data['total_liquidity'] = data['bid_qty'] + data['ask_qty']\n data['trade_imbalance'] = data['buy_qty'] - data['sell_qty']\n data['total_trades'] = data['buy_qty'] + data['sell_qty']\n \n # Volume-based features\n data['volume_per_trade'] = data['volume'] / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n data['buy_volume_ratio'] = data['buy_qty'] / (data['volume'] + 1e-8)\n data['sell_volume_ratio'] = data['sell_qty'] / (data['volume'] + 1e-8)\n \n # Pressure and imbalance features\n data['buying_pressure'] = data['buy_qty'] / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n data['selling_pressure'] = data['sell_qty'] / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n \n data['order_imbalance'] = (data['bid_qty'] - data['ask_qty']) / (data['bid_qty'] + data['ask_qty'] + 1e-8)\n data['order_imbalance_abs'] = np.abs(data['order_imbalance'])\n data['bid_liquidity_ratio'] = data['bid_qty'] / (data['volume'] + 1e-8)\n data['ask_liquidity_ratio'] = data['ask_qty'] / (data['volume'] + 1e-8)\n data['depth_imbalance'] = data['total_trades'] - data['volume']\n \n # Ratio features\n data['buy_sell_ratio'] = data['buy_qty'] / (data['sell_qty'] + 1e-8)\n data['bid_ask_ratio'] = data['bid_qty'] / (data['ask_qty'] + 1e-8)\n data['volume_liquidity_ratio'] = data['volume'] / (data['bid_qty'] + data['ask_qty'] + 1e-8)\n\n # Product features\n data['buy_volume_product'] = data['buy_qty'] * data['volume']\n data['sell_volume_product'] = data['sell_qty'] * data['volume']\n data['bid_ask_product'] = data['bid_qty'] * data['ask_qty']\n \n # Competition and activity features\n data['market_competition'] = (data['buy_qty'] * data['sell_qty']) / ((data['buy_qty'] + data['sell_qty']) + 1e-8)\n data['liquidity_competition'] = (data['bid_qty'] * data['ask_qty']) / ((data['bid_qty'] + data['ask_qty']) + 1e-8)\n \n total_activity = data['buy_qty'] + data['sell_qty'] + data['bid_qty'] + data['ask_qty']\n data['market_activity'] = total_activity\n data['activity_concentration'] = data['volume'] / (total_activity + 1e-8)\n \n # Advanced microstructure features\n data['info_arrival_rate'] = (data['buy_qty'] + data['sell_qty']) / (data['volume'] + 1e-8)\n data['market_making_intensity'] = (data['bid_qty'] + data['ask_qty']) / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n data['effective_spread_proxy'] = np.abs(data['buy_qty'] - data['sell_qty']) / (data['volume'] + 1e-8)\n \n # Exponentially weighted moving average of order flow imbalance\n lambda_decay = 0.95\n ofi = data['buy_qty'] - data['sell_qty']\n data['order_flow_imbalance_ewm'] = ofi.ewm(alpha=1-lambda_decay).mean()\n\n # Clean up infinite values\n data = data.replace([np.inf, -np.inf], np.nan)\n \n final_features = len(data.columns)\n new_features = final_features - initial_features\n \n print(f\" โœ… Added {new_features} engineered features\")\n print(f\" ๐Ÿ“Š Total features: {initial_features} โ†’ {final_features}\")\n \n logger.info(f\"Feature engineering for {dataset_name}: added {new_features} features ({initial_features} โ†’ {final_features})\")\n \n return data\n \n def load_and_merge_datasets(self) -> Tuple[pd.DataFrame, pd.DataFrame]:\n \"\"\"\n Load and merge multiple datasets efficiently based on configuration.\n \n Returns:\n Tuple of (merged_train_df, merged_test_df)\n \"\"\"\n print_header(\"FLEXIBLE DATA LOADING AND MERGING\")\n start_time = time.time()\n \n try:\n # Load and process train datasets\n print_step(\"Loading and merging train datasets\")\n merged_train = self._load_and_merge_dataset_group(\n self.config.TRAIN_DATASETS, \"TRAIN\"\n )\n \n # Load and process test datasets \n print_step(\"Loading and merging test datasets\")\n merged_test = self._load_and_merge_dataset_group(\n self.config.TEST_DATASETS, \"TEST\"\n )\n \n # Apply feature selection\n print_step(\"Applying feature selection\")\n merged_train, merged_test = self._apply_feature_selection(merged_train, merged_test)\n \n # Final memory optimization after merging and feature selection\n print_step(\"Final memory optimization\")\n merged_train = self.reduce_mem_usage(merged_train, \"Final Merged Train\")\n merged_test = self.reduce_mem_usage(merged_test, \"Final Merged Test\")\n \n # Validate final datasets\n print_step(\"Validating final merged datasets\")\n self._validate_final_datasets(merged_train, merged_test)\n \n # Display final dataset information\n print_step(\"Final dataset information\")\n print(f\" ๐Ÿ“Š Train shape: {merged_train.shape}\")\n print(f\" ๐Ÿ“Š Test shape: {merged_test.shape}\")\n print(f\" ๐Ÿ“Š Train columns: {list(merged_train.columns)}\")\n print(f\" ๐Ÿ“Š Test columns: {list(merged_test.columns)}\")\n print(f\" ๐ŸŽฏ Target column '{self.config.TARGET_COLUMN}' present in train: {self.config.TARGET_COLUMN in merged_train.columns}\")\n \n # Comprehensive memory cleanup after data loading\n print_step(\"Post-processing memory cleanup\")\n print_memory_usage(\"before data cleanup\")\n \n # Clean up any temporary variables from data loading\n variables_to_cleanup = [\n 'loaded_datasets', 'dataset_names', 'dataset_info', 'dataset_df',\n 'raw_df', 'processed_df', 'before_shape', 'after_shape'\n ]\n \n # Multiple garbage collection passes\n collected_total = 0\n for i in range(3):\n collected = gc.collect()\n collected_total += collected\n if collected > 0:\n print(f\" ๐Ÿ—‘๏ธ GC pass {i+1}: {collected} objects collected\")\n \n print(f\" โœ… Total cleanup: {collected_total} objects collected\")\n print_memory_usage(\"after data cleanup\")\n \n total_time = time.time() - start_time\n print_result(\"Total loading time\", total_time, \".1f\")\n print(\"โœจ Data loading and merging completed successfully!\")\n logger.info(f\"Data loading completed in {total_time:.1f}s - Train: {merged_train.shape}, Test: {merged_test.shape}\")\n \n return merged_train, merged_test\n \n except Exception as e:\n print_error(f\"Data loading failed: {str(e)}\")\n logger.error(f\"Data loading failed: {str(e)}\")\n raise\n \n def _load_and_merge_dataset_group(self, dataset_configs: List[DatasetConfig], \n group_name: str) -> pd.DataFrame:\n \"\"\"\n Load and merge a group of datasets (either train or test).\n \n Args:\n dataset_configs: List of DatasetConfig objects\n group_name: Name for logging (e.g., \"TRAIN\", \"TEST\")\n \n Returns:\n Merged dataframe\n \"\"\"\n print(f\"\\n ๐Ÿš€ Processing {group_name} dataset group ({len(dataset_configs)} files)\")\n \n loaded_datasets = {}\n \n # Step 1: Load individual datasets\n total_files = len(dataset_configs)\n for i, dataset_config in enumerate(dataset_configs, 1):\n print_progress(i, total_files, \"files\")\n print(f\" ๐Ÿ“‚ Loading {dataset_config.dataset_name}...\")\n print(f\" ๐Ÿ“ Path: {dataset_config.file_path}\")\n \n try:\n load_start = time.time()\n \n # Check if file exists\n if not os.path.exists(dataset_config.file_path):\n if dataset_config.is_required:\n raise FileNotFoundError(f\"Required file not found: {dataset_config.file_path}\")\n else:\n print_warning(f\"Optional file not found, skipping: {dataset_config.file_path}\")\n continue\n \n # Load the dataset (supports both CSV and Parquet)\n file_extension = dataset_config.file_path.lower().split('.')[-1]\n try:\n if file_extension == 'parquet':\n raw_df = pd.read_parquet(dataset_config.file_path)\n file_type = \"Parquet\"\n elif file_extension in ['csv', 'tsv']:\n # For CSV, try to automatically parse datetime columns\n raw_df = pd.read_csv(dataset_config.file_path)\n file_type = \"CSV\"\n \n # Try to detect and parse datetime columns\n datetime_cols_detected = []\n for col in raw_df.columns:\n # Check if column name suggests it's a datetime\n col_lower = col.lower()\n if any(pattern in col_lower for pattern in ['timestamp', 'time', 'date', 'datetime', 'ts']):\n try:\n raw_df[col] = pd.to_datetime(raw_df[col])\n datetime_cols_detected.append(col)\n except:\n pass # If conversion fails, keep original type\n \n if datetime_cols_detected:\n print(f\" ๐Ÿ“… Auto-detected datetime columns: {datetime_cols_detected}\")\n \n else:\n # Try CSV as default\n print_warning(f\"Unknown file extension '.{file_extension}', trying CSV format\")\n raw_df = pd.read_csv(dataset_config.file_path)\n file_type = \"CSV (default)\"\n except Exception as file_error:\n if file_extension == 'parquet':\n print_warning(f\"Failed to read as Parquet, trying CSV: {str(file_error)}\")\n raw_df = pd.read_csv(dataset_config.file_path)\n file_type = \"CSV (fallback)\"\n else:\n raise file_error\n \n load_time = time.time() - load_start\n \n print(f\" โœ… Raw shape: {raw_df.shape} loaded in {load_time:.1f}s ({file_type})\")\n \n # DETAILED COLUMN AND INDEX ANALYSIS\n print(f\" ๐Ÿ“‹ Regular columns ({len(raw_df.columns)}): {list(raw_df.columns)[:10]}...\")\n \n # Check if any required ID column is in the index\n if hasattr(raw_df.index, 'name') and raw_df.index.name:\n print(f\" ๐Ÿ“… Index column: {raw_df.index.name} (type: {raw_df.index.dtype})\")\n \n # Check if the index name matches any of our required ID columns (regardless of data type)\n index_name = raw_df.index.name\n if index_name in dataset_config.id_columns:\n print(f\" ๐Ÿ”„ Index '{index_name}' is a required ID column - resetting index\")\n raw_df = raw_df.reset_index()\n print(f\" โœ… After reset_index: shape={raw_df.shape}, columns={list(raw_df.columns)[:10]}...\")\n else:\n print(f\" โ„น๏ธ Index '{index_name}' is not in required ID columns: {dataset_config.id_columns}\")\n \n print(f\" ๐Ÿ“Š Final columns after index handling: {len(raw_df.columns)}\")\n print(f\" ๐Ÿ“‹ All columns: {list(raw_df.columns)}\")\n \n # Check for datetime columns in regular columns\n datetime_cols = [col for col in raw_df.columns if pd.api.types.is_datetime64_any_dtype(raw_df[col])]\n if datetime_cols:\n print(f\" ๐Ÿ“… Datetime columns found: {datetime_cols}\")\n \n # Verify required ID columns are now present\n missing_id_cols_check = [col for col in dataset_config.id_columns if col not in raw_df.columns]\n if missing_id_cols_check:\n print(f\" โš ๏ธ Still missing ID columns after index reset: {missing_id_cols_check}\")\n else:\n print(f\" โœ… All required ID columns found: {dataset_config.id_columns}\")\n \n # Analyze columns for debugging if there might be issues\n if not all(col in raw_df.columns for col in dataset_config.id_columns):\n analyze_dataframe_columns(raw_df, f\"{dataset_config.dataset_name} (Raw)\")\n \n # Process columns according to configuration\n processed_df = self._process_dataset_columns(raw_df, dataset_config)\n \n # Apply memory optimization\n processed_df = self.reduce_mem_usage(processed_df, dataset_config.dataset_name)\n \n # Apply feature engineering if enabled\n processed_df = self.add_features(processed_df, dataset_config.dataset_name)\n \n loaded_datasets[dataset_config.dataset_name] = {\n 'dataframe': processed_df,\n 'config': dataset_config\n }\n \n print(f\" โœ… Final shape: {processed_df.shape}\")\n print(f\" ๐Ÿ“Š Final columns: {list(processed_df.columns)[:10]}...\" if len(processed_df.columns) > 10 else f\" ๐Ÿ“Š Final columns: {list(processed_df.columns)}\")\n \n except Exception as e:\n if dataset_config.is_required:\n print_error(f\"Failed to load required dataset '{dataset_config.dataset_name}': {str(e)}\")\n raise\n else:\n print_warning(f\"Failed to load optional dataset '{dataset_config.dataset_name}': {str(e)}\")\n continue\n \n if not loaded_datasets:\n raise ValueError(f\"No datasets successfully loaded for {group_name} group\")\n \n # Step 2: Merge datasets\n print(f\"\\n ๐Ÿ”— Merging {len(loaded_datasets)} {group_name} datasets...\")\n \n # Start with the first dataset\n dataset_names = list(loaded_datasets.keys())\n merged_df = loaded_datasets[dataset_names[0]]['dataframe'].copy()\n merge_info = [f\"Base: {dataset_names[0]} ({merged_df.shape})\"]\n \n # Merge remaining datasets\n for dataset_name in dataset_names[1:]:\n dataset_info = loaded_datasets[dataset_name]\n dataset_df = dataset_info['dataframe']\n merge_columns = dataset_info['config'].id_columns\n \n print(f\" ๐Ÿ”— Merging {dataset_name} on {merge_columns}...\")\n \n # Validate merge columns exist\n missing_cols_left = [col for col in merge_columns if col not in merged_df.columns]\n missing_cols_right = [col for col in merge_columns if col not in dataset_df.columns]\n \n if missing_cols_left:\n raise KeyError(f\"Merge columns {missing_cols_left} not found in merged dataset\")\n if missing_cols_right:\n raise KeyError(f\"Merge columns {missing_cols_right} not found in {dataset_name}\")\n \n # Perform merge\n before_shape = merged_df.shape\n merged_df = pd.merge(\n merged_df, \n dataset_df, \n on=merge_columns, \n how='inner'\n )\n after_shape = merged_df.shape\n \n merge_info.append(f\"+ {dataset_name}: {before_shape} โ†’ {after_shape}\")\n print(f\" โœ… Merge result: {before_shape} โ†’ {after_shape}\")\n \n print(f\"\\n ๐Ÿ“Š {group_name} merge summary:\")\n for info in merge_info:\n print(f\" {info}\")\n \n print(f\" โœ… Final {group_name} dataset: {merged_df.shape}\")\n logger.info(f\"{group_name} datasets merged successfully: {merged_df.shape}\")\n \n return merged_df\n \n def _process_dataset_columns(self, df: pd.DataFrame, \n config: DatasetConfig) -> pd.DataFrame:\n \"\"\"\n Process dataset columns according to configuration.\n ALWAYS preserves ID columns regardless of their data type.\n \n Args:\n df: Raw dataframe\n config: Dataset configuration\n \n Returns:\n Processed dataframe with only desired columns\n \"\"\"\n # Validate that ID columns exist\n missing_id_cols = [col for col in config.id_columns if col not in df.columns]\n if missing_id_cols:\n print(f\" โŒ Missing ID columns: {missing_id_cols}\")\n print(f\" ๐Ÿ“‹ Available columns: {list(df.columns)[:20]}{'...' if len(df.columns) > 20 else ''}\")\n print(f\" ๐Ÿ“Š Total columns: {len(df.columns)}\")\n \n # Check for similar column names\n similar_cols = []\n for missing_col in missing_id_cols:\n for available_col in df.columns:\n if missing_col.lower() in available_col.lower() or available_col.lower() in missing_col.lower():\n similar_cols.append((missing_col, available_col))\n \n if similar_cols:\n print(f\" ๐Ÿ’ก Similar columns found: {similar_cols}\")\n print(f\" ๐Ÿ’ก Consider updating your DatasetConfig id_columns\")\n \n # Check for datetime columns that might have been parsed\n datetime_cols = [col for col in df.columns if pd.api.types.is_datetime64_any_dtype(df[col])]\n if datetime_cols:\n print(f\" ๐Ÿ“… Available datetime columns: {datetime_cols}\")\n \n # Check for object columns that might be string timestamps\n object_cols = [col for col in df.columns if df[col].dtype == 'object']\n potential_timestamp_cols = [col for col in object_cols if any(pattern in col.lower() \n for pattern in ['timestamp', 'time', 'date', 'datetime', 'ts'])]\n if potential_timestamp_cols:\n print(f\" ๐Ÿ• Potential string timestamp columns: {potential_timestamp_cols}\")\n \n raise KeyError(f\"ID columns {missing_id_cols} not found in {config.dataset_name}. \"\n f\"Available columns: {list(df.columns)}\")\n \n # ALWAYS preserve ID columns regardless of configuration\n columns_to_keep = set(config.id_columns)\n \n # Initialize existing_feature_cols to prevent UnboundLocalError\n existing_feature_cols = []\n \n # Add feature columns if specified\n if config.feature_columns:\n # Validate that specified feature columns exist\n missing_feature_cols = [col for col in config.feature_columns if col not in df.columns]\n if missing_feature_cols:\n print_warning(f\"Feature columns {missing_feature_cols} not found in {config.dataset_name}\")\n \n # Keep only existing feature columns\n existing_feature_cols = [col for col in config.feature_columns if col in df.columns]\n columns_to_keep.update(existing_feature_cols)\n \n print(f\" ๐ŸŽฏ Feature columns specified: {len(config.feature_columns)}\")\n print(f\" โœ… Feature columns found: {len(existing_feature_cols)}\")\n else:\n # Keep all columns (ID columns already included)\n columns_to_keep.update(df.columns)\n \n # Convert to list and select columns\n columns_to_keep = list(columns_to_keep)\n processed_df = df[columns_to_keep].copy()\n \n print(f\" ๐Ÿ“‹ Kept {len(columns_to_keep)} columns from {len(df.columns)} available\")\n print(f\" ๐Ÿ”— ID columns preserved: {config.id_columns}\")\n \n # Show data types of ID columns for debugging\n for id_col in config.id_columns:\n if id_col in processed_df.columns:\n print(f\" ๐Ÿ“ ID column '{id_col}' type: {processed_df[id_col].dtype}\")\n \n return processed_df\n \n def _apply_feature_selection(self, train_df: pd.DataFrame, test_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:\n \"\"\"Apply feature selection based on SELECTED_FEATURES.\"\"\"\n if not self.config.SELECTED_FEATURES:\n print(\" โ„น๏ธ No feature selection specified, keeping all features\")\n return train_df, test_df\n \n print(f\" ๐ŸŽฏ Applying feature selection: {len(self.config.SELECTED_FEATURES)} features\")\n \n # Identify which columns to keep (selected features + required columns)\n required_cols = {self.config.TARGET_COLUMN, self.config.ID_COLUMN, self.config.TIMESTAMP_COLUMN}\n required_cols = {col for col in required_cols if col} # Remove None values\n \n # For train data: keep target + ID/timestamp + selected features\n train_cols_to_keep = []\n for col in required_cols:\n if col in train_df.columns:\n train_cols_to_keep.append(col)\n \n # Add selected features that exist in train data\n for feature in self.config.SELECTED_FEATURES:\n if feature in train_df.columns and feature not in train_cols_to_keep:\n train_cols_to_keep.append(feature)\n elif feature not in train_df.columns:\n print_warning(f\"Selected feature '{feature}' not found in train data\")\n \n # For test data: keep ID + selected features (no target)\n test_cols_to_keep = []\n for col in [self.config.ID_COLUMN, self.config.TIMESTAMP_COLUMN]:\n if col and col in test_df.columns:\n test_cols_to_keep.append(col)\n \n # Add selected features that exist in test data\n for feature in self.config.SELECTED_FEATURES:\n if feature in test_df.columns and feature not in test_cols_to_keep:\n test_cols_to_keep.append(feature)\n elif feature not in test_df.columns:\n print_warning(f\"Selected feature '{feature}' not found in test data\")\n \n # Apply selection\n train_selected = train_df[train_cols_to_keep].copy()\n test_selected = test_df[test_cols_to_keep].copy()\n \n print(f\" โœ… Train features: {train_df.shape[1]} โ†’ {train_selected.shape[1]} columns\")\n print(f\" โœ… Test features: {test_df.shape[1]} โ†’ {test_selected.shape[1]} columns\")\n \n return train_selected, test_selected\n \n def _validate_final_datasets(self, train_df: pd.DataFrame, test_df: pd.DataFrame):\n \"\"\"Validate that final merged datasets have required columns.\"\"\"\n \n # Check for target column in train data\n if self.config.TARGET_COLUMN and self.config.TARGET_COLUMN not in train_df.columns:\n available_cols = list(train_df.columns)\n raise KeyError(f\"Target column '{self.config.TARGET_COLUMN}' not found in merged train data. \"\n f\"Available columns: {available_cols}\")\n \n # Check that test data has ID column if specified\n if self.config.ID_COLUMN and self.config.ID_COLUMN not in test_df.columns:\n available_cols = list(test_df.columns)\n raise KeyError(f\"ID column '{self.config.ID_COLUMN}' not found in test data. \"\n f\"Available columns: {available_cols}\")\n \n print(f\" โœ… All required columns validated\")\n if self.config.TARGET_COLUMN:\n print(f\" ๐ŸŽฏ Target column: {self.config.TARGET_COLUMN}\")\n if self.config.ID_COLUMN:\n print(f\" ๐Ÿ†” ID column: {self.config.ID_COLUMN}\")\n\nclass ModelTrainer:\n \"\"\"Handle individual model training with different data subsets and algorithms.\"\"\"\n \n def __init__(self, config: ModelConfig):\n self.config = config\n self.weight_calculator = WeightCalculator()\n print_step(\"Model Trainer Initialized\")\n print(f\" ๐ŸŽฏ Number of model configurations: {len(self.config.MODEL_CONFIGS)}\")\n print(f\" ๐Ÿ“Š Cross-validation folds: {self.config.N_FOLDS}\")\n print(f\" ๐Ÿ”ฅ Multi-GPU training: {self.config.USE_MULTI_GPU}\")\n if self.config.USE_MULTI_GPU:\n print(f\" ๐ŸŽฎ GPU devices: {self.config.GPU_DEVICES}\")\n print(f\" โšก Parallel fold training: {self.config.PARALLEL_FOLD_TRAINING}\")\n logger.info(f\"ModelTrainer initialized with {len(self.config.MODEL_CONFIGS)} model configs\")\n \n def _get_gpu_device_for_fold(self, fold_idx: int) -> int:\n \"\"\"Get GPU device ID for a specific fold.\"\"\"\n if not self.config.USE_MULTI_GPU:\n return 0\n return self.config.GPU_DEVICES[fold_idx % len(self.config.GPU_DEVICES)]\n \n def _prepare_gpu_params(self, base_params: Dict[str, Any], gpu_device: int) -> Dict[str, Any]:\n \"\"\"Prepare GPU-specific parameters for XGBoost/LightGBM.\"\"\"\n params = base_params.copy()\n \n if 'tree_method' in params: # XGBoost\n if self.config.USE_MULTI_GPU:\n params['device'] = f'cuda:{gpu_device}'\n params['tree_method'] = 'gpu_hist'\n else:\n params['tree_method'] = 'hist'\n \n if 'device' in params and params['device'] == 'gpu': # LightGBM\n if self.config.USE_MULTI_GPU:\n params['device'] = f'gpu'\n params['gpu_device_id'] = gpu_device\n else:\n params['device'] = 'cpu'\n \n return params\n \n def _train_single_fold(self, fold_data: Dict[str, Any]) -> Tuple[np.ndarray, np.ndarray, float]:\n \"\"\"\n Train a single fold on a specific GPU device.\n \n Args:\n fold_data: Dictionary containing fold training data and parameters\n \n Returns:\n Tuple of (fold_oof_predictions, fold_test_predictions, fold_score)\n \"\"\"\n fold_idx = fold_data['fold_idx']\n X_fold_train = fold_data['X_fold_train']\n y_fold_train = fold_data['y_fold_train']\n X_fold_valid = fold_data['X_fold_valid']\n y_fold_valid = fold_data['y_fold_valid']\n X_test = fold_data['X_test']\n fold_weights = fold_data['fold_weights']\n algorithm = fold_data['algorithm']\n model_name = fold_data['model_name']\n gpu_device = fold_data['gpu_device']\n \n print(f\" ๐ŸŽฎ Fold {fold_idx + 1} training on GPU {gpu_device}\")\n \n # Prepare GPU-specific parameters\n if algorithm == 'xgb':\n params = self._prepare_gpu_params(self.config.XGB_PARAMS, gpu_device)\n model = XGBRegressor(**params)\n else: # lgb\n params = self._prepare_gpu_params(self.config.LGBM_PARAMS, gpu_device)\n model = LGBMRegressor(**params)\n \n # Train model\n try:\n if algorithm == 'xgb':\n model.fit(\n X_fold_train, y_fold_train,\n sample_weight=fold_weights,\n eval_set=[(X_fold_valid, y_fold_valid)],\n early_stopping_rounds=25,\n verbose=False\n )\n else: # lgb\n model.fit(\n X_fold_train, y_fold_train,\n sample_weight=fold_weights,\n eval_set=[(X_fold_valid, y_fold_valid)],\n callbacks=[lgb.early_stopping(25), lgb.log_evaluation(0)]\n )\n \n # Make predictions\n valid_preds = model.predict(X_fold_valid)\n test_preds = model.predict(X_test)\n \n # Calculate fold score\n fold_score = pearsonr(y_fold_valid, valid_preds)[0]\n \n print(f\" โœ… Fold {fold_idx + 1} completed on GPU {gpu_device}: {fold_score:.6f}\")\n \n # Quick cleanup after fold completion\n del model\n gc.collect()\n \n return valid_preds, test_preds, fold_score\n \n except Exception as e:\n print(f\" โŒ Error in fold {fold_idx + 1} on GPU {gpu_device}: {str(e)}\")\n raise\n \n def train_model_ensemble(self, train_df: pd.DataFrame, test_df: pd.DataFrame) -> Dict[str, Any]:\n \"\"\"\n Train ensemble of models with different data subsets and algorithms.\n \n Args:\n train_df: Training dataframe\n test_df: Test dataframe\n \n Returns:\n Dictionary containing OOF predictions, test predictions, and scores\n \"\"\"\n print_header(\"MODEL ENSEMBLE TRAINING\")\n start_time = time.time()\n \n # Prepare features and target\n feature_cols = [col for col in train_df.columns \n if col not in {self.config.TARGET_COLUMN, self.config.ID_COLUMN, self.config.TIMESTAMP_COLUMN}]\n \n print_step(\"Feature preparation\")\n print(f\" ๐ŸŽฏ Feature columns: {len(feature_cols)}\")\n print(f\" ๐Ÿ“‹ Features: {feature_cols}\")\n \n X_train = train_df[feature_cols].copy()\n y_train = train_df[self.config.TARGET_COLUMN].copy()\n X_test = test_df[feature_cols].copy()\n \n print(f\" ๐Ÿ“Š Training data shape: {X_train.shape}\")\n print(f\" ๐Ÿ“Š Test data shape: {X_test.shape}\")\n print(f\" ๐Ÿ“Š Target range: [{y_train.min():.4f}, {y_train.max():.4f}]\")\n \n # COMPREHENSIVE MEMORY CLEANUP BEFORE MODEL TRAINING\n print_step(\"Pre-training memory cleanup\")\n print_memory_usage(\"before training cleanup\")\n \n # Clear temporary variables from feature preparation\n variables_to_cleanup = [\n 'feature_cols', 'train_df', 'test_df'\n ]\n \n # Delete original dataframes to free memory (we have copies)\n del train_df, test_df\n \n # Force multiple garbage collection passes\n print(\" ๐Ÿงน Performing thorough memory cleanup...\")\n collected_total = 0\n for i in range(4): # Extra passes before training\n collected = gc.collect()\n collected_total += collected\n if collected > 0:\n print(f\" ๐Ÿ—‘๏ธ GC pass {i+1}: {collected} objects collected\")\n \n print(f\" โœ… Pre-training cleanup: {collected_total} objects collected\")\n print_memory_usage(\"after training cleanup\")\n print(\" ๐Ÿš€ Memory optimized for model training!\")\n \n # Initialize cross-validation (matching reference implementation)\n kf = KFold(n_splits=self.config.N_FOLDS, shuffle=False)\n \n # Initialize prediction storage\n model_results = {}\n \n # Train XGBoost models with different data subsets\n print_step(\"Training XGBoost models\")\n print_memory_usage(\"XGBoost training start\")\n full_dataset_oof_xgb = None\n \n for model_config in self.config.MODEL_CONFIGS:\n model_name = f\"XGB_{model_config['name'].replace(' ', '_').replace('(', '').replace(')', '')}\"\n print_step(f\"Training {model_name}\")\n \n oof_preds, test_preds, score = self._train_single_model(\n X_train, y_train, X_test, kf, \n model_config['percent'], 'xgb', model_name\n )\n \n # Store full dataset OOF for filling subset models\n if model_config['percent'] == 1.0:\n full_dataset_oof_xgb = oof_preds.copy()\n \n model_results[model_name] = {\n 'oof_preds': oof_preds,\n 'test_preds': test_preds,\n 'score': score,\n 'algorithm': 'XGBoost',\n 'data_percent': model_config['percent']\n }\n \n # Quick memory cleanup after each model\n gc.collect()\n \n # Train LightGBM models with different data subsets\n print_step(\"Training LightGBM models\")\n print_memory_usage(\"LightGBM training start\")\n full_dataset_oof_lgb = None\n \n for model_config in self.config.MODEL_CONFIGS:\n model_name = f\"LGB_{model_config['name'].replace(' ', '_').replace('(', '').replace(')', '')}\"\n print_step(f\"Training {model_name}\")\n \n oof_preds, test_preds, score = self._train_single_model(\n X_train, y_train, X_test, kf, \n model_config['percent'], 'lgb', model_name\n )\n \n # Store full dataset OOF for filling subset models\n if model_config['percent'] == 1.0:\n full_dataset_oof_lgb = oof_preds.copy()\n \n model_results[model_name] = {\n 'oof_preds': oof_preds,\n 'test_preds': test_preds,\n 'score': score,\n 'algorithm': 'LightGBM',\n 'data_percent': model_config['percent']\n }\n \n # Quick memory cleanup after each model\n gc.collect()\n \n # Fill missing OOF predictions for subset models using full dataset predictions\n print_step(\"Filling missing OOF predictions for subset models\")\n for model_name, results in model_results.items():\n if results['data_percent'] < 1.0:\n cutoff_idx = int(len(X_train) * (1 - results['data_percent']))\n if 'XGB' in model_name and full_dataset_oof_xgb is not None:\n # Fill samples before cutoff with full dataset XGB predictions\n mask = results['oof_preds'][:cutoff_idx] == 0\n results['oof_preds'][:cutoff_idx][mask] = full_dataset_oof_xgb[:cutoff_idx][mask]\n elif 'LGB' in model_name and full_dataset_oof_lgb is not None:\n # Fill samples before cutoff with full dataset LGB predictions\n mask = results['oof_preds'][:cutoff_idx] == 0\n results['oof_preds'][:cutoff_idx][mask] = full_dataset_oof_lgb[:cutoff_idx][mask]\n \n # Add target for ensemble creation\n model_results['y_true'] = y_train\n \n # Final memory cleanup and reporting\n print_step(\"Final training memory cleanup\")\n collected = gc.collect()\n if collected > 0:\n print(f\" ๐Ÿ—‘๏ธ Final cleanup: {collected} objects collected\")\n print_memory_usage(\"training complete\")\n \n total_time = time.time() - start_time\n print_result(\"Total training time\", total_time, \".1f\")\n print(\"โœจ Model ensemble training completed successfully!\")\n logger.info(f\"Model ensemble training completed in {total_time:.1f}s with {len(model_results)-1} models\")\n \n return model_results\n \n def _train_single_model(self, X_train: pd.DataFrame, y_train: pd.Series, X_test: pd.DataFrame,\n kf: KFold, data_percent: float, algorithm: str, model_name: str) -> Tuple[np.ndarray, np.ndarray, float]:\n \"\"\"\n Train a single model with specified data percentage and algorithm.\n \n Args:\n X_train: Training features\n y_train: Training target\n X_test: Test features\n kf: KFold object for cross-validation\n data_percent: Percentage of recent data to use\n algorithm: 'xgb' or 'lgb'\n model_name: Name for logging\n \n Returns:\n Tuple of (oof_predictions, test_predictions, cv_score)\n \"\"\"\n print(f\" ๐ŸŽฏ Model: {model_name} ({algorithm.upper()}) - {data_percent*100:.0f}% recent data\")\n \n # Calculate data cutoff\n cutoff_idx = int(len(X_train) * (1 - data_percent)) if data_percent < 1.0 else 0\n \n if cutoff_idx > 0:\n print(f\" ๐Ÿ“Š Using samples {cutoff_idx} to {len(X_train)-1} ({len(X_train) - cutoff_idx:,} samples)\")\n train_subset = X_train.iloc[cutoff_idx:].reset_index(drop=True)\n target_subset = y_train.iloc[cutoff_idx:].reset_index(drop=True)\n else:\n print(f\" ๐Ÿ“Š Using all {len(X_train):,} samples\")\n train_subset = X_train.copy()\n target_subset = y_train.copy()\n \n # Create time weights for the subset\n sample_weights = self.weight_calculator.create_time_weights(\n len(train_subset), self.config.DECAY_FACTOR\n )\n \n # Initialize prediction arrays\n oof_preds = np.zeros(len(y_train))\n test_preds = np.zeros(len(X_test))\n cv_scores = []\n \n # Prepare fold data for parallel training\n fold_data_list = []\n for fold, (train_idx, valid_idx) in enumerate(kf.split(train_subset)):\n gpu_device = self._get_gpu_device_for_fold(fold)\n \n fold_data = {\n 'fold_idx': fold,\n 'train_idx': train_idx,\n 'valid_idx': valid_idx,\n 'X_fold_train': train_subset.iloc[train_idx],\n 'y_fold_train': target_subset.iloc[train_idx],\n 'X_fold_valid': train_subset.iloc[valid_idx],\n 'y_fold_valid': target_subset.iloc[valid_idx],\n 'X_test': X_test,\n 'fold_weights': sample_weights[train_idx],\n 'algorithm': algorithm,\n 'model_name': model_name,\n 'gpu_device': gpu_device\n }\n fold_data_list.append(fold_data)\n \n # Train folds in parallel if enabled, otherwise sequentially\n if self.config.PARALLEL_FOLD_TRAINING and self.config.USE_MULTI_GPU and len(self.config.GPU_DEVICES) > 1:\n print(f\" โšก Training {self.config.N_FOLDS} folds in parallel across {len(self.config.GPU_DEVICES)} GPUs\")\n \n # Use ThreadPoolExecutor for GPU parallelism (better for GPU workloads)\n with ThreadPoolExecutor(max_workers=len(self.config.GPU_DEVICES)) as executor:\n # Submit all fold training jobs\n future_to_fold = {executor.submit(self._train_single_fold, fold_data): fold_data['fold_idx'] \n for fold_data in fold_data_list}\n \n # Collect results as they complete\n fold_results = {}\n for future in as_completed(future_to_fold):\n fold_idx = future_to_fold[future]\n try:\n valid_preds, fold_test_preds, fold_score = future.result()\n fold_results[fold_idx] = {\n 'valid_preds': valid_preds,\n 'test_preds': fold_test_preds,\n 'score': fold_score\n }\n cv_scores.append(fold_score)\n print(f\" โœ… Fold {fold_idx + 1} completed: {fold_score:.6f}\")\n except Exception as e:\n print(f\" โŒ Fold {fold_idx + 1} failed: {str(e)}\")\n raise\n \n # Process results in fold order\n for fold_idx in range(self.config.N_FOLDS):\n result = fold_results[fold_idx]\n valid_preds = result['valid_preds']\n fold_test_preds = result['test_preds']\n \n # Get the original fold validation indices from fold_data_list\n original_valid_idx = fold_data_list[fold_idx]['valid_idx']\n \n # Store out-of-fold predictions with proper index mapping\n if cutoff_idx > 0:\n # Map subset validation indices back to full dataset indices\n full_dataset_valid_idx = original_valid_idx + cutoff_idx\n oof_preds[full_dataset_valid_idx] = valid_preds\n else:\n oof_preds[original_valid_idx] = valid_preds\n \n # Accumulate test predictions\n test_preds += fold_test_preds\n else:\n # Sequential training (fallback)\n print(f\" ๐Ÿ”„ Training {self.config.N_FOLDS} folds sequentially\")\n for fold_data in fold_data_list:\n fold_idx = fold_data['fold_idx']\n print(f\" ๐Ÿ”„ Fold {fold_idx + 1}/{self.config.N_FOLDS}\")\n \n valid_preds, fold_test_preds, fold_score = self._train_single_fold(fold_data)\n \n # Store out-of-fold predictions\n valid_idx = fold_data['valid_idx']\n if cutoff_idx > 0:\n # Map subset validation indices back to full dataset indices\n full_dataset_valid_idx = valid_idx + cutoff_idx\n oof_preds[full_dataset_valid_idx] = valid_preds\n else:\n oof_preds[valid_idx] = valid_preds\n \n # Accumulate test predictions\n test_preds += fold_test_preds\n cv_scores.append(fold_score)\n \n # Average test predictions\n test_preds /= self.config.N_FOLDS\n \n # Calculate overall CV score\n cv_score = pearsonr(y_train, oof_preds)[0]\n print(f\" ๐ŸŽฏ CV Score: {cv_score:.6f} (ยฑ{np.std(cv_scores):.6f})\")\n \n return oof_preds, test_preds, cv_score\n\nclass EnsembleManager:\n \"\"\"Manage ensemble creation and evaluation.\"\"\"\n \n def __init__(self, config: ModelConfig):\n self.config = config\n print_step(\"Ensemble Manager Initialized\")\n logger.info(\"EnsembleManager initialized\")\n \n def create_ensemble_predictions(self, model_results: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"\n Create ensemble predictions using different strategies.\n \n Args:\n model_results: Dictionary containing individual model results\n \n Returns:\n Dictionary containing ensemble results\n \"\"\"\n print_header(\"ENSEMBLE CREATION AND EVALUATION\")\n \n # Extract individual model results\n y_true = model_results['y_true']\n individual_models = {k: v for k, v in model_results.items() if k != 'y_true'}\n \n print_step(\"Individual model performance\")\n for model_name, results in individual_models.items():\n print(f\" ๐ŸŽฏ {model_name}: {results['score']:.6f} ({results['algorithm']}, {results['data_percent']*100:.0f}% data)\")\n \n # Simple average ensemble\n print_step(\"Creating simple average ensemble\")\n avg_oof = np.mean([results['oof_preds'] for results in individual_models.values()], axis=0)\n avg_test = np.mean([results['test_preds'] for results in individual_models.values()], axis=0)\n avg_score = pearsonr(y_true, avg_oof)[0]\n print_result(\"Average ensemble score\", avg_score)\n \n # Weighted ensemble (performance-based)\n print_step(\"Creating performance-weighted ensemble\")\n scores = np.array([results['score'] for results in individual_models.values()])\n weights = scores / scores.sum()\n \n weighted_oof = np.average([results['oof_preds'] for results in individual_models.values()], \n weights=weights, axis=0)\n weighted_test = np.average([results['test_preds'] for results in individual_models.values()], \n weights=weights, axis=0)\n weighted_score = pearsonr(y_true, weighted_oof)[0]\n print_result(\"Weighted ensemble score\", weighted_score)\n \n print(\" ๐Ÿ“Š Model weights:\")\n for i, (model_name, weight) in enumerate(zip(individual_models.keys(), weights)):\n print(f\" {model_name}: {weight:.4f}\")\n \n # Create per-learner ensembles first (matching reference implementation)\n print_step(\"Creating per-learner ensembles\")\n learner_ensembles = {}\n \n # Separate models by algorithm\n xgb_models = {k: v for k, v in individual_models.items() if 'XGB' in k}\n lgb_models = {k: v for k, v in individual_models.items() if 'LGB' in k}\n \n # Create XGBoost ensemble\n if xgb_models:\n xgb_oof_simple = np.mean([v['oof_preds'] for v in xgb_models.values()], axis=0)\n xgb_test_simple = np.mean([v['test_preds'] for v in xgb_models.values()], axis=0)\n xgb_score_simple = pearsonr(y_true, xgb_oof_simple)[0]\n print(f\" ๐ŸŽฏ XGBoost simple ensemble score: {xgb_score_simple:.6f}\")\n learner_ensembles['xgb'] = {\n 'oof_simple': xgb_oof_simple,\n 'test_simple': xgb_test_simple\n }\n \n # Create LightGBM ensemble\n if lgb_models:\n lgb_oof_simple = np.mean([v['oof_preds'] for v in lgb_models.values()], axis=0)\n lgb_test_simple = np.mean([v['test_preds'] for v in lgb_models.values()], axis=0)\n lgb_score_simple = pearsonr(y_true, lgb_oof_simple)[0]\n print(f\" ๐ŸŽฏ LightGBM simple ensemble score: {lgb_score_simple:.6f}\")\n learner_ensembles['lgb'] = {\n 'oof_simple': lgb_oof_simple,\n 'test_simple': lgb_test_simple\n }\n \n # Final ensemble creation based on strategy\n print_step(\"Creating final ensemble\")\n \n if self.config.ENSEMBLE_STRATEGY == \"individual_models\":\n print(\" ๐ŸŽฏ Using individual model weights strategy\")\n final_oof, final_test, ensemble_type = self._create_individual_weighted_ensemble(individual_models, y_true)\n elif self.config.ENSEMBLE_STRATEGY == \"performance_based\":\n print(\" ๐ŸŽฏ Using performance-based ensemble strategy\")\n final_oof, final_test, ensemble_type = self._create_performance_based_ensemble(individual_models, y_true)\n else:\n print(\" ๐ŸŽฏ Using learner-level ensemble strategy\")\n final_oof, final_test, ensemble_type = self._create_learner_level_ensemble(learner_ensembles)\n \n final_score = pearsonr(y_true, final_oof)[0]\n \n # For backward compatibility, also compute the old ensemble scores\n avg_score = pearsonr(y_true, avg_oof)[0]\n \n # Prepare results summary based on ensemble strategy\n results_summary = self._prepare_results_summary(\n individual_models, learner_ensembles, final_score, ensemble_type, \n weights, xgb_score_simple if 'xgb' in learner_ensembles else None,\n lgb_score_simple if 'lgb' in learner_ensembles else None\n )\n \n ensemble_results = {\n 'final_predictions': final_test,\n 'final_oof': final_oof,\n 'final_score': final_score,\n 'ensemble_type': ensemble_type,\n 'results_summary': results_summary,\n 'individual_models': individual_models\n }\n \n print_result(\"Final ensemble score\", final_score)\n print(\"โœจ Ensemble creation completed successfully!\")\n logger.info(f\"Ensemble created successfully with score: {final_score:.6f}\")\n \n return ensemble_results\n \n def _create_individual_weighted_ensemble(self, individual_models: Dict[str, Any], y_true: np.ndarray) -> Tuple[np.ndarray, np.ndarray, str]:\n \"\"\"\n Create ensemble using individual model weights.\n \n Args:\n individual_models: Dictionary of individual model results\n y_true: True target values for scoring\n \n Returns:\n Tuple of (final_oof, final_test, ensemble_type)\n \"\"\"\n print(\" ๐Ÿ” Creating individual model weighted ensemble\")\n \n # Get weights and normalize if needed\n model_weights = self.config.INDIVIDUAL_MODEL_WEIGHTS.copy()\n total_weight = sum(model_weights.values())\n \n if abs(total_weight - 1.0) > 1e-6:\n print_warning(f\"Individual model weights sum to {total_weight:.6f}, not 1.0. Normalizing weights.\")\n for model_name in model_weights:\n model_weights[model_name] /= total_weight\n ensemble_type = \"individual_weighted_normalized\"\n print(\" โœ… Using normalized individual model weights\")\n else:\n ensemble_type = \"individual_weighted\"\n print(\" โœ… Using individual model weights\")\n \n # Display weights\n print(\" ๐Ÿ“Š Individual model weights:\")\n for model_name, weight in model_weights.items():\n if model_name in individual_models:\n algorithm = individual_models[model_name]['algorithm']\n data_percent = individual_models[model_name]['data_percent']\n print(f\" {model_name}: {weight:.4f} ({algorithm}, {data_percent*100:.0f}% data)\")\n \n # Create weighted ensemble\n weighted_oof = np.zeros(len(y_true))\n weighted_test = np.zeros(len(individual_models[list(individual_models.keys())[0]]['test_preds']))\n \n for model_name, weight in model_weights.items():\n if model_name in individual_models:\n weighted_oof += weight * individual_models[model_name]['oof_preds']\n weighted_test += weight * individual_models[model_name]['test_preds']\n else:\n print_warning(f\"Model {model_name} not found in individual_models, skipping\")\n \n score = pearsonr(y_true, weighted_oof)[0]\n print(f\" ๐ŸŽฏ Individual weighted ensemble score: {score:.6f}\")\n \n return weighted_oof, weighted_test, ensemble_type\n \n def _create_learner_level_ensemble(self, learner_ensembles: Dict[str, Any]) -> Tuple[np.ndarray, np.ndarray, str]:\n \"\"\"\n Create ensemble using learner-level weights (XGBoost vs LightGBM).\n \n Args:\n learner_ensembles: Dictionary of learner ensemble results\n \n Returns:\n Tuple of (final_oof, final_test, ensemble_type)\n \"\"\"\n print(\" ๐Ÿ” Creating learner-level ensemble\")\n \n # Check if custom weights are provided\n if self.config.CUSTOM_ENSEMBLE_WEIGHTS and len(self.config.CUSTOM_ENSEMBLE_WEIGHTS) > 0:\n custom_weights = self.config.CUSTOM_ENSEMBLE_WEIGHTS\n \n # Validate custom weights\n if len(custom_weights) != len(learner_ensembles):\n print_warning(f\"Custom weights length ({len(custom_weights)}) doesn't match number of learners ({len(learner_ensembles)}). Using equal weights.\")\n final_oof = np.mean([le['oof_simple'] for le in learner_ensembles.values()], axis=0)\n final_test = np.mean([le['test_simple'] for le in learner_ensembles.values()], axis=0)\n ensemble_type = \"simple_across_learners\"\n print(\" โœ… Using simple average ensemble across learners\")\n elif abs(sum(custom_weights) - 1.0) > 1e-6:\n print_warning(f\"Custom weights sum to {sum(custom_weights):.6f}, not 1.0. Normalizing weights.\")\n # Normalize weights to sum to 1.0\n custom_weights = [w / sum(custom_weights) for w in custom_weights]\n \n # Apply custom weights\n learner_oofs = [le['oof_simple'] for le in learner_ensembles.values()]\n learner_tests = [le['test_simple'] for le in learner_ensembles.values()]\n \n final_oof = np.average(learner_oofs, weights=custom_weights, axis=0)\n final_test = np.average(learner_tests, weights=custom_weights, axis=0)\n ensemble_type = \"custom_weighted_across_learners_normalized\"\n \n print(\" โœ… Using custom weighted ensemble across learners (normalized)\")\n print(f\" ๐Ÿ“Š Normalized weights: {custom_weights}\")\n else:\n # Apply custom weights\n learner_oofs = [le['oof_simple'] for le in learner_ensembles.values()]\n learner_tests = [le['test_simple'] for le in learner_ensembles.values()]\n \n final_oof = np.average(learner_oofs, weights=custom_weights, axis=0)\n final_test = np.average(learner_tests, weights=custom_weights, axis=0)\n ensemble_type = \"custom_weighted_across_learners\"\n \n print(\" โœ… Using custom weighted ensemble across learners\")\n print(f\" ๐Ÿ“Š Custom weights: {custom_weights}\")\n \n # Show weight assignment\n for i, (learner_name, weight) in enumerate(zip(learner_ensembles.keys(), custom_weights)):\n algorithm = \"XGBoost\" if learner_name == 'xgb' else \"LightGBM\"\n print(f\" {algorithm}: {weight:.4f}\")\n else:\n # Default: simple average ensemble\n final_oof = np.mean([le['oof_simple'] for le in learner_ensembles.values()], axis=0)\n final_test = np.mean([le['test_simple'] for le in learner_ensembles.values()], axis=0)\n ensemble_type = \"simple_across_learners\"\n print(\" โœ… Using simple average ensemble across learners (default)\")\n \n return final_oof, final_test, ensemble_type\n \n def _create_performance_based_ensemble(self, individual_models: Dict[str, Any], y_true: np.ndarray) -> Tuple[np.ndarray, np.ndarray, str]:\n \"\"\"\n Create ensemble using performance-based weights (automatically calculated from CV scores).\n \n Args:\n individual_models: Dictionary of individual model results\n y_true: True target values for scoring\n \n Returns:\n Tuple of (final_oof, final_test, ensemble_type)\n \"\"\"\n print(\" ๐Ÿ” Creating performance-based weighted ensemble\")\n \n # Extract scores and create performance-based weights\n model_names = list(individual_models.keys())\n scores = np.array([individual_models[model]['score'] for model in model_names])\n \n print(f\" ๐Ÿ“Š Individual model scores:\")\n for model_name, score in zip(model_names, scores):\n algorithm = individual_models[model_name]['algorithm']\n data_percent = individual_models[model_name]['data_percent']\n print(f\" {model_name}: {score:.6f} ({algorithm}, {data_percent*100:.0f}% data)\")\n \n # Calculate performance-based weights (higher score = higher weight)\n # Using softmax-like transformation to convert scores to weights\n # Add small epsilon to prevent division by zero\n epsilon = 1e-8\n exp_scores = np.exp((scores - scores.max()) / 0.1) # Temperature scaling for smoother weights\n weights = exp_scores / (exp_scores.sum() + epsilon)\n \n # Alternative: Simple proportional weights\n # weights = scores / scores.sum()\n \n print(f\" โš–๏ธ Performance-based weights:\")\n for model_name, weight, score in zip(model_names, weights, scores):\n print(f\" {model_name}: {weight:.4f} (score: {score:.6f})\")\n \n print(f\" ๐Ÿ“Š Weight statistics:\")\n print(f\" Min weight: {weights.min():.4f}\")\n print(f\" Max weight: {weights.max():.4f}\")\n print(f\" Weight range: {weights.max() - weights.min():.4f}\")\n print(f\" Weight sum: {weights.sum():.6f}\")\n \n # Create weighted ensemble\n weighted_oof = np.zeros(len(y_true))\n weighted_test = np.zeros(len(individual_models[model_names[0]]['test_preds']))\n \n for model_name, weight in zip(model_names, weights):\n weighted_oof += weight * individual_models[model_name]['oof_preds']\n weighted_test += weight * individual_models[model_name]['test_preds']\n \n # Calculate ensemble score\n ensemble_score = pearsonr(y_true, weighted_oof)[0]\n print(f\" ๐ŸŽฏ Performance-based ensemble score: {ensemble_score:.6f}\")\n \n # Compare with simple average\n simple_avg_oof = np.mean([individual_models[model]['oof_preds'] for model in model_names], axis=0)\n simple_avg_score = pearsonr(y_true, simple_avg_oof)[0]\n improvement = ensemble_score - simple_avg_score\n print(f\" ๐Ÿ“ˆ Improvement over simple average: {improvement:+.6f}\")\n \n ensemble_type = \"performance_based_weighted\"\n print(\" โœ… Performance-based ensemble created successfully\")\n \n return weighted_oof, weighted_test, ensemble_type\n \n def _prepare_results_summary(self, individual_models: Dict[str, Any], learner_ensembles: Dict[str, Any], \n final_score: float, ensemble_type: str, weights: np.ndarray,\n xgb_score_simple: Optional[float], lgb_score_simple: Optional[float]) -> List[Dict[str, Any]]:\n \"\"\"\n Prepare results summary based on ensemble strategy.\n \n Args:\n individual_models: Dictionary of individual model results\n learner_ensembles: Dictionary of learner ensemble results \n final_score: Final ensemble score\n ensemble_type: Type of ensemble used\n weights: Performance-based weights for backward compatibility\n xgb_score_simple: XGBoost ensemble score (if available)\n lgb_score_simple: LightGBM ensemble score (if available)\n \n Returns:\n List of result dictionaries for summary\n \"\"\"\n results_summary = []\n \n # Add individual model results with appropriate weights\n for model_name, results in individual_models.items():\n if self.config.ENSEMBLE_STRATEGY == \"individual_models\":\n # Use individual model weights if specified\n weight_in_ensemble = self.config.INDIVIDUAL_MODEL_WEIGHTS.get(model_name, 0.0)\n # Normalize if needed\n total_weight = sum(self.config.INDIVIDUAL_MODEL_WEIGHTS.values())\n if abs(total_weight - 1.0) > 1e-6:\n weight_in_ensemble /= total_weight\n elif self.config.ENSEMBLE_STRATEGY == \"performance_based\":\n # Calculate performance-based weight for this model\n scores_array = np.array([individual_models[m]['score'] for m in individual_models.keys()])\n model_idx = list(individual_models.keys()).index(model_name)\n # Use same calculation as in _create_performance_based_ensemble\n epsilon = 1e-8\n exp_scores = np.exp((scores_array - scores_array.max()) / 0.1)\n weights_array = exp_scores / (exp_scores.sum() + epsilon)\n weight_in_ensemble = weights_array[model_idx]\n else:\n # Use equal weight for learner-level strategy\n weight_in_ensemble = 1.0 / len(individual_models)\n \n results_summary.append({\n 'model': model_name,\n 'algorithm': results['algorithm'],\n 'data_percent': results['data_percent'],\n 'pearson_correlation': results['score'],\n 'weight_in_ensemble': weight_in_ensemble\n })\n \n # Add learner ensemble results if using learner-level strategy\n if self.config.ENSEMBLE_STRATEGY == \"learner_level\":\n # Determine learner ensemble weights\n ensemble_weights = []\n if self.config.CUSTOM_ENSEMBLE_WEIGHTS and len(self.config.CUSTOM_ENSEMBLE_WEIGHTS) == len(learner_ensembles):\n # Use custom weights (potentially normalized)\n if abs(sum(self.config.CUSTOM_ENSEMBLE_WEIGHTS) - 1.0) > 1e-6:\n ensemble_weights = [w / sum(self.config.CUSTOM_ENSEMBLE_WEIGHTS) for w in self.config.CUSTOM_ENSEMBLE_WEIGHTS]\n else:\n ensemble_weights = self.config.CUSTOM_ENSEMBLE_WEIGHTS\n else:\n # Use equal weights\n ensemble_weights = [1.0 / len(learner_ensembles)] * len(learner_ensembles)\n \n weight_idx = 0\n if 'xgb' in learner_ensembles and xgb_score_simple is not None:\n results_summary.append({\n 'model': 'XGBoost Simple Ensemble',\n 'algorithm': 'XGBoost Ensemble',\n 'data_percent': 1.0,\n 'pearson_correlation': xgb_score_simple,\n 'weight_in_ensemble': ensemble_weights[weight_idx]\n })\n weight_idx += 1\n \n if 'lgb' in learner_ensembles and lgb_score_simple is not None:\n results_summary.append({\n 'model': 'LightGBM Simple Ensemble',\n 'algorithm': 'LightGBM Ensemble',\n 'data_percent': 1.0,\n 'pearson_correlation': lgb_score_simple,\n 'weight_in_ensemble': ensemble_weights[weight_idx]\n })\n \n # Add final ensemble result\n ensemble_name = f\"Final Ensemble ({ensemble_type.replace('_', ' ').title()})\"\n results_summary.append({\n 'model': ensemble_name,\n 'algorithm': 'Final Ensemble',\n 'data_percent': 1.0,\n 'pearson_correlation': final_score,\n 'weight_in_ensemble': 1.0\n })\n \n return results_summary\n\nclass XGBoostLightGBMPipeline:\n \"\"\"Main pipeline orchestrator for XGBoost and LightGBM ensemble modeling.\"\"\"\n \n def __init__(self, config: ModelConfig):\n self.config = config\n print_header(\"XGBOOST-LIGHTGBM ENSEMBLE PIPELINE\")\n print(f\" ๐ŸŽฏ Pipeline initialized with {len(self.config.TRAIN_DATASETS)} train and {len(self.config.TEST_DATASETS)} test datasets\")\n print(f\" ๐Ÿค– Model algorithms: XGBoost + LightGBM\")\n print(f\" ๐Ÿ“Š Cross-validation folds: {self.config.N_FOLDS}\")\n print(f\" ๐ŸŽฏ Selected features: {len(self.config.SELECTED_FEATURES)}\")\n print(f\" โš–๏ธ Ensemble strategy: {self.config.ENSEMBLE_STRATEGY}\")\n \n # Validate configuration\n is_valid, error_msg = self.config.validate_weights()\n if not is_valid:\n raise ValueError(f\"Configuration validation failed: {error_msg}\")\n \n if self.config.ENSEMBLE_STRATEGY == \"individual_models\":\n print(f\" ๐ŸŽฏ Individual model weights specified: {len(self.config.INDIVIDUAL_MODEL_WEIGHTS)} models\")\n elif self.config.ENSEMBLE_STRATEGY == \"performance_based\":\n print(f\" ๐ŸŽฏ Performance-based ensemble: weights calculated from CV scores\")\n elif self.config.CUSTOM_ENSEMBLE_WEIGHTS:\n print(f\" ๐ŸŽฏ Custom learner weights: {self.config.CUSTOM_ENSEMBLE_WEIGHTS}\")\n else:\n print(f\" ๐ŸŽฏ Using equal weights for ensemble\")\n \n # Initialize components\n self.data_processor = DataProcessor(config)\n self.model_trainer = ModelTrainer(config)\n self.ensemble_manager = EnsembleManager(config)\n \n logger.info(\"XGBoostLightGBMPipeline initialized successfully\")\n \n def run_pipeline(self) -> Dict[str, Any]:\n \"\"\"\n Run the complete pipeline from data loading to final predictions.\n \n Returns:\n Dictionary containing final results and metadata\n \"\"\"\n pipeline_start = time.time()\n print(f\"โฐ Pipeline start time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n print_memory_usage(\"pipeline start\")\n \n try:\n # Step 1: Load and merge datasets\n train_df, test_df = self.data_processor.load_and_merge_datasets()\n \n # Step 2: Train model ensemble\n model_results = self.model_trainer.train_model_ensemble(train_df, test_df)\n \n # Step 3: Create ensemble predictions\n ensemble_results = self.ensemble_manager.create_ensemble_predictions(model_results)\n \n # Step 4: Save results\n self._save_results(ensemble_results, test_df)\n \n # Final summary\n total_time = time.time() - pipeline_start\n print_header(\"PIPELINE COMPLETED SUCCESSFULLY\")\n print_result(\"Total pipeline time\", total_time, \".1f\")\n print_result(\"Final ensemble score\", ensemble_results['final_score'])\n print_memory_usage(\"pipeline end\")\n print(f\" ๐Ÿ“ Submission saved to: {self.config.SUBMISSION_FILENAME}\")\n print(f\" ๐Ÿ“ Results saved to: {self.config.RESULTS_FILENAME}\")\n print(\"๐ŸŽ‰ Pipeline execution completed successfully!\")\n \n logger.info(f\"Pipeline completed successfully in {total_time:.1f}s with final score: {ensemble_results['final_score']:.6f}\")\n \n # Get final shapes before cleanup\n train_shape = train_df.shape if 'train_df' in locals() else \"N/A\"\n test_shape = test_df.shape if 'test_df' in locals() else \"N/A\"\n \n return {\n 'ensemble_results': ensemble_results,\n 'execution_time': total_time,\n 'train_shape': train_shape,\n 'test_shape': test_shape\n }\n \n except Exception as e:\n print_error(f\"Pipeline failed: {str(e)}\")\n logger.error(f\"Pipeline failed: {str(e)}\")\n raise\n \n def _save_results(self, ensemble_results: Dict[str, Any], test_df: pd.DataFrame):\n \"\"\"Save predictions and results to files.\"\"\"\n print_step(\"Saving results to files\")\n \n try:\n # Save submission file\n if self.config.ID_COLUMN in test_df.columns:\n submission_df = pd.DataFrame({\n self.config.ID_COLUMN: test_df[self.config.ID_COLUMN],\n 'prediction': ensemble_results['final_predictions']\n })\n else:\n submission_df = pd.DataFrame({\n 'ID': range(len(ensemble_results['final_predictions'])),\n 'prediction': ensemble_results['final_predictions']\n })\n \n submission_df.to_csv(self.config.SUBMISSION_FILENAME, index=False)\n print(f\" โœ… Submission saved: {self.config.SUBMISSION_FILENAME}\")\n \n # Save detailed results\n results_df = pd.DataFrame(ensemble_results['results_summary'])\n results_df.to_csv(self.config.RESULTS_FILENAME, index=False)\n print(f\" โœ… Results saved: {self.config.RESULTS_FILENAME}\")\n \n # Display submission preview\n print(\" ๐Ÿ“‹ Submission preview:\")\n print(submission_df.head().to_string(index=False))\n \n # Display results summary\n print(\" ๐Ÿ“‹ Results summary:\")\n print(results_df.to_string(index=False))\n \n except Exception as e:\n print_error(f\"Failed to save results: {str(e)}\")\n raise","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Multiple File Merging\n\n```python\nconfig = ModelConfig(\n TRAIN_DATASETS=[\n DatasetConfig(\n file_path=\"features_part1.parquet\", # Parquet for faster loading\n feature_columns=[\"X1\", \"X2\", \"X3\"],\n id_columns=[\"timestamp\"],\n dataset_name=\"Features Part 1\"\n ),\n DatasetConfig(\n file_path=\"features_part2.csv\", # CSV also supported\n feature_columns=[\"X4\", \"X5\", \"X6\"],\n id_columns=[\"timestamp\"],\n dataset_name=\"Features Part 2\"\n ),\n DatasetConfig(\n file_path=\"targets.tsv\", # TSV files supported too\n feature_columns=[\"label\"],\n id_columns=[\"timestamp\"],\n dataset_name=\"Targets\"\n )\n ],\n # ... rest of configuration\n)\n```","metadata":{}},{"cell_type":"markdown","source":"### Custom Ensemble Strategy\n\n```python\nconfig = ModelConfig(\n MODEL_CONFIGS=[\n {\"name\": \"Full Dataset\", \"percent\": 1.00, \"priority\": 1},\n {\"name\": \"Recent 90%\", \"percent\": 0.90, \"priority\": 2},\n {\"name\": \"Recent 70%\", \"percent\": 0.70, \"priority\": 3},\n {\"name\": \"Recent 50%\", \"percent\": 0.50, \"priority\": 4},\n {\"name\": \"Recent 30%\", \"percent\": 0.30, \"priority\": 5},\n ],\n # ... rest of configuration\n)\n```","metadata":{}},{"cell_type":"markdown","source":"### Multi-GPU Configuration\n\n```python\nconfig = ModelConfig(\n # Enable multi-GPU training\n USE_MULTI_GPU=True,\n GPU_DEVICES=[0, 1], # Use both T4 GPUs\n PARALLEL_FOLD_TRAINING=True,\n \n # XGBoost parameters for GPU\n XGB_PARAMS={\n \"tree_method\": \"gpu_hist\", # GPU-accelerated training\n \"device\": \"cuda:0\", # Will be automatically set per fold\n # ... other parameters\n },\n \n # LightGBM parameters for GPU \n LGBM_PARAMS={\n \"device\": \"gpu\", # Enable GPU training\n \"gpu_device_id\": 0, # Will be automatically set per fold\n # ... other parameters\n },\n \n # ... rest of configuration\n)\n```","metadata":{}},{"cell_type":"markdown","source":"**GPU Training Benefits:**\n- **2x Speed Improvement**: Parallel fold training across both GPUs\n- **Automatic Load Balancing**: Folds distributed evenly across available GPUs \n- **Memory Efficiency**: Each GPU handles subset of folds independently\n- **Fault Tolerance**: Graceful fallback to sequential training if needed\n\n**GPU Configuration Options:**\n- `USE_MULTI_GPU=True`: Enable multi-GPU support\n- `GPU_DEVICES=[0, 1]`: Specify which GPU devices to use\n- `PARALLEL_FOLD_TRAINING=True`: Train folds in parallel (vs sequential)\n- Automatic GPU parameter adjustment for both XGBoost and LightGBM\n\n### Memory Management\n\nThe pipeline includes comprehensive memory management throughout the entire process:\n\n```python\n# Automatic memory cleanup is built-in, but you can monitor it:\npipeline = XGBoostLightGBMPipeline(config)\nresults = pipeline.run_pipeline() # Memory monitored throughout\n```\n\n**Memory Management Features:**\n- **Pre-training Cleanup**: Comprehensive memory cleanup before model training starts\n- **Real-time Monitoring**: Memory usage tracking at all pipeline stages\n- **Automatic Garbage Collection**: Multiple GC passes for thorough cleanup\n- **Variable Cleanup**: Deletion of temporary objects and variables\n- **Per-model Cleanup**: Memory cleanup after each model training\n- **GPU Memory Optimization**: Smart memory management for multi-GPU training\n\n**Memory Tracking Stages:**\n- Pipeline start and end\n- Data loading completion \n- Pre-training cleanup\n- XGBoost/LightGBM training start\n- Training completion\n- Real-time memory usage reporting\n\n**Memory Benefits:**\n- Reduced peak memory usage during training\n- Better GPU memory utilization\n- Faster training through optimized memory access\n- Prevention of out-of-memory errors\n- Automatic cleanup of temporary variables\n\n## Output Files\n\nThe pipeline generates several output files:\n\n1. **Submission File** (`submission_ensemble_XGB_LGB.csv`): Final predictions\n2. **Results File** (`ensemble_results.csv`): Detailed model performance metrics\n3. **Console Output**: Real-time progress and performance information\n\n## Time-Based Weighting\n\nThe pipeline uses exponential decay weighting to emphasize recent samples:\n\n```python\ndef create_time_weights(n_samples, decay_factor=0.95):\n \"\"\"\n Create exponentially decaying weights based on sample position.\n More recent samples (higher indices) get higher weights.\n \"\"\"\n positions = np.arange(n_samples)\n normalized_positions = positions / (n_samples - 1)\n weights = decay_factor ** (1 - normalized_positions)\n weights = weights * n_samples / weights.sum()\n return weights\n```\n\n- `decay_factor=0.95`: 5% decay per time unit\n- `decay_factor=0.98`: 2% decay per time unit (stronger recent emphasis)\n\n## Cross-Validation Strategy\n\nThe pipeline uses KFold cross-validation with proper handling of different data subsets:\n\n1. **Full Dataset Models**: Use all available training data\n2. **Subset Models**: Use only the most recent X% of data\n3. **OOF Predictions**: Properly map predictions back to original indices\n4. **Ensemble Creation**: Combine predictions using simple average or performance weighting\n\n## Memory Optimization\n\nThe pipeline includes automatic memory optimization that can reduce RAM usage by 50-80%:\n\n```python\nconfig = ModelConfig(\n REDUCE_MEMORY_USAGE=True, # Enable memory optimization\n # ... other parameters\n)\n```\n\n**How it works:**\n- Automatically converts `int64` โ†’ `int8/int16/int32` where possible\n- Automatically converts `float64` โ†’ `float16/float32` where possible\n- Maintains numerical precision within safe ranges\n- Provides detailed logging of memory savings\n\n## Feature Engineering\n\nGenerate 30+ market microstructure features from basic market data:\n\n```python\nconfig = ModelConfig(\n ADD_ENGINEERED_FEATURES=True, # Enable feature engineering\n # ... other parameters\n)\n```\n\n**Requirements:** Your data must contain: `bid_qty`, `ask_qty`, `buy_qty`, `sell_qty`, `volume`\n\n**Generated features include:**\n- **Market Microstructure**: bid-ask spread, liquidity measures, trade imbalances\n- **Volume-Based**: volume per trade, buy/sell volume ratios\n- **Pressure Indicators**: buying/selling pressure, order imbalances\n- **Activity Measures**: market competition, activity concentration\n- **Advanced Features**: order flow imbalance EWMA, market making intensity\n\n## Performance Tips\n\n### GPU Acceleration\n```python\nxgb_params = {\n \"tree_method\": \"gpu_hist\", # Use GPU\n \"device\": \"cuda\",\n # ... other parameters\n}\n```\n\n### Memory Optimization\n- Enable `REDUCE_MEMORY_USAGE=True` for automatic optimization\n- Use appropriate `max_bin` settings for XGBoost\n- Monitor memory usage during large dataset processing\n- Consider data chunking for very large datasets\n\n### Feature Selection\n- Start with a subset of most important features\n- Use the `SELECTED_FEATURES` parameter to control feature usage\n- Enable `ADD_ENGINEERED_FEATURES=True` for market data\n- Monitor training time vs. performance trade-offs\n\n## Troubleshooting\n\n### Common Issues\n\n1. **File Not Found**: Check file paths in `DatasetConfig`\n2. **Memory Issues**: Reduce `n_estimators` or use smaller data subsets\n3. **GPU Issues**: Fall back to `tree_method=\"hist\"` for CPU usage\n4. **Merge Failures**: Ensure ID columns exist in all datasets\n5. **Feature Mismatches**: Check that selected features exist in data files\n\n### Debug Mode\n\nEnable verbose logging:\n```python\nimport logging\nlogging.basicConfig(level=logging.DEBUG)\n```\n\n## Model Performance\n\nThe pipeline trains multiple model variants:\n\n1. **XGBoost Models**: Full dataset + multiple recent data subsets\n2. **LightGBM Models**: Full dataset + multiple recent data subsets \n3. **Ensemble**: Combines all individual models using:\n - Simple average ensemble\n - Performance-weighted ensemble (chooses best)","metadata":{}},{"cell_type":"markdown","source":"# Example Usage: One can add their own features and columns to get a baseline from XGB and LGBM using time decays. The params are to be adjusted according to the data provided.","metadata":{}},{"cell_type":"code","source":"# Example usage\nif __name__ == \"__main__\":\n # User-provided parameters\n lgbm_params = {\n \"boosting_type\": \"gbdt\",\n \"colsample_bytree\": 0.5625888953382505,\n \"learning_rate\": 0.029312951475451557,\n \"min_child_samples\": 63,\n \"min_child_weight\": 0.11456572852335424,\n \"n_estimators\": 150,\n \"n_jobs\": -1,\n \"num_leaves\": 37,\n \"random_state\": 42,\n \"reg_alpha\": 85.2476527854083,\n \"reg_lambda\": 99.38305361388907,\n \"subsample\": 0.450669817684892,\n \"verbose\": -1\n }\n\n xgb_params = {\n \"tree_method\": \"gpu_hist\",\n \"colsample_bylevel\": 0.4778015829774066,\n \"colsample_bynode\": 0.362764358742407,\n \"colsample_bytree\": 0.7107423488010493,\n \"gamma\": 1.7094857725240398,\n \"learning_rate\": 0.02213323588455387,\n \"max_depth\": 20,\n \"max_leaves\": 12,\n \"min_child_weight\": 16,\n \"n_estimators\": 1800,\n \"n_jobs\": -1,\n \"random_state\": 42,\n \"reg_alpha\": 39.352415706891264,\n \"reg_lambda\": 75.44843704068275,\n \"subsample\": 0.06566669853471274,\n \"verbosity\": 0\n }\n\n base_features = [\n \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n ]\n\n additional_features = [\n \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n \"X888\", \"X421\", \"X333\"\n ]\n \n selected_features = base_features + additional_features\n \n # Create configuration\n config = ModelConfig(\n # Dataset configurations - UPDATE THESE PATHS AS NEEDED\n TRAIN_DATASETS=[\n DatasetConfig(\n file_path=\"/kaggle/input/drw-crypto-market-prediction/train.parquet\",\n feature_columns=selected_features + [\"label\"], # Will keep these + ID columns\n id_columns=[\"timestamp\"],\n dataset_name=\"Train Features\",\n is_required=True\n ),\n DatasetConfig(\n file_path=\"/kaggle/input/drw-all-auxiliary-datasets/train_regimes.csv\", # Adjust filename\n feature_columns=[\"predicted_regime_Est\"], # Additional regime feature\n id_columns=[\"timestamp\"],\n dataset_name=\"Train Regimes\",\n is_required=True\n ),\n DatasetConfig(\n file_path=\"/kaggle/input/drw-all-auxiliary-datasets/pca_train_dataset.csv\", # Adjust filename\n feature_columns=[\"pca_0\",\"pca_1\",\"pca_2\",\"pca_3\",\"pca_4\",\"pca_5\",\"pca_6\",\"pca_7\",\"pca_8\",\"pca_9\"], # Additional pca features\n id_columns=[\"timestamp\"],\n dataset_name=\"Train PCA\",\n is_required=True\n )\n ],\n TEST_DATASETS=[\n DatasetConfig(\n file_path=\"/kaggle/input/drw-crypto-market-prediction/test.parquet\",\n feature_columns=selected_features, # No label in test data\n id_columns=[\"ID\"],\n dataset_name=\"Test Features\",\n is_required=True\n ),\n DatasetConfig(\n file_path=\"/kaggle/input/drw-all-auxiliary-datasets/test_regimes.csv\",\n feature_columns=[\"predicted_regime_Est\"], # Additional regime feature\n id_columns=[\"ID\"],\n dataset_name=\"Test Regimes\",\n is_required=True\n ),\n DatasetConfig(\n file_path=\"/kaggle/input/drw-all-auxiliary-datasets/pca_test_dataset.csv\",\n feature_columns=[\"pca_0\",\"pca_1\",\"pca_2\",\"pca_3\",\"pca_4\",\"pca_5\",\"pca_6\",\"pca_7\",\"pca_8\",\"pca_9\"], # Additional pca features\n id_columns=[\"ID\"],\n dataset_name=\"Test PCA\",\n is_required=True\n )\n ],\n \n # Column configuration\n TARGET_COLUMN=\"label\", # Target column from main train dataset\n ID_COLUMN=\"ID\", # ID column for test predictions\n TIMESTAMP_COLUMN=\"timestamp\", # Timestamp column for merging\n \n # Model parameters\n XGB_PARAMS=xgb_params,\n LGBM_PARAMS=lgbm_params,\n SELECTED_FEATURES=selected_features + [\"predicted_regime_Est\"], # Include regime feature\n \n # Multi-GPU settings\n USE_MULTI_GPU=True,\n GPU_DEVICES=[0, 1], # Use both T4 GPUs\n PARALLEL_FOLD_TRAINING=True,\n REDUCE_MEMORY_USAGE=True, # Optimize memory usage\n ADD_ENGINEERED_FEATURES=True,\n \n # Ensemble weight configuration (choose one strategy):\n \n # STRATEGY 1: Learner-level weights (XGBoost vs LightGBM ensembles)\n # ENSEMBLE_STRATEGY=\"learner_level\",\n # CUSTOM_ENSEMBLE_WEIGHTS=[0.6, 0.4], # Give XGBoost 60% weight, LightGBM 40% weight\n \n # STRATEGY 2: Individual model weights (granular control over all 6 models)\n # ENSEMBLE_STRATEGY=\"individual_models\",\n # INDIVIDUAL_MODEL_WEIGHTS={\n # # XGBoost models\n # \"XGB_Full_Dataset_100%\": 0.25, # XGBoost full dataset\n # \"XGB_Recent_Data_75%\": 0.20, # XGBoost 75% recent data\n # \"XGB_Recent_Data_50%\": 0.15, # XGBoost 50% recent data\n # # LightGBM models \n # \"LGB_Full_Dataset_100%\": 0.20, # LightGBM full dataset\n # \"LGB_Recent_Data_75%\": 0.15, # LightGBM 75% recent data\n # \"LGB_Recent_Data_50%\": 0.05, # LightGBM 50% recent data\n # },\n # Weights will be automatically normalized if they don't sum to 1.0\n \n # STRATEGY 3: Performance-based weights (automatic weight calculation)\n ENSEMBLE_STRATEGY=\"performance_based\",\n # No additional configuration needed - weights calculated from CV scores\n # Higher performing models automatically get higher weights\n \n # Alternative individual weights example (equal weights):\n # INDIVIDUAL_MODEL_WEIGHTS={\n # \"XGB_Full_Dataset_100%\": 1/6, \"XGB_Recent_Data_75%\": 1/6, \"XGB_Recent_Data_50%\": 1/6,\n # \"LGB_Full_Dataset_100%\": 1/6, \"LGB_Recent_Data_75%\": 1/6, \"LGB_Recent_Data_50%\": 1/6,\n # },\n \n # Alternative individual weights example (prefer full dataset models):\n # INDIVIDUAL_MODEL_WEIGHTS={\n # \"XGB_Full_Dataset_100%\": 0.35, \"XGB_Recent_Data_75%\": 0.15, \"XGB_Recent_Data_50%\": 0.10,\n # \"LGB_Full_Dataset_100%\": 0.25, \"LGB_Recent_Data_75%\": 0.10, \"LGB_Recent_Data_50%\": 0.05,\n # },\n \n # Other settings\n N_FOLDS=5,\n RANDOM_STATE=42,\n DECAY_FACTOR=0.95,\n \n SUBMISSION_FILENAME=\"submission.csv\",\n )\n \n # Run pipeline\n pipeline = XGBoostLightGBMPipeline(config)\n results = pipeline.run_pipeline()\n \n print(f\"\\n Pipeline completed! Final ensemble score: {results['ensemble_results']['final_score']:.6f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}