{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:05:27.465913Z","iopub.execute_input":"2025-06-04T16:05:27.4661Z","iopub.status.idle":"2025-06-04T16:05:29.171802Z","shell.execute_reply.started":"2025-06-04T16:05:27.466084Z","shell.execute_reply":"2025-06-04T16:05:29.171015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# INITIAL SETUP AND PACKAGE INSTALLATION\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction Pipeline - Setup and Global Configuration\nThis module provides the foundation for all prediction models with dynamic model discovery\n\"\"\"\n\nimport subprocess\nimport sys\nimport os\nimport gc\nimport warnings\nimport json\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nfrom typing import List, Dict, Tuple, Optional, Any\nfrom datetime import datetime\n\nwarnings.filterwarnings('ignore')\n\n# Install only essential base packages\nprint(\"Installing base packages...\")\nbase_packages = [\"pandas\", \"numpy\", \"scipy\", \"scikit-learn\"]\nsubprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\"] + base_packages + [\"--quiet\"])\n\n# Import base packages after installation\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr\nfrom sklearn.model_selection import train_test_split, KFold, TimeSeriesSplit\nfrom sklearn.linear_model import Ridge\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom sklearn.metrics import mean_absolute_error\n\n# Memory management function\ndef aggressive_memory_cleanup():\n    \"\"\"Aggressively clean up memory to prevent kernel death\"\"\"\n    # PyTorch cleanup\n    if 'torch' in sys.modules:\n        try:\n            import torch\n            if torch.cuda.is_available():\n                torch.cuda.empty_cache()\n                torch.cuda.synchronize()\n        except:\n            pass\n    \n    # TensorFlow cleanup\n    if 'tensorflow' in sys.modules:\n        try:\n            import tensorflow as tf\n            tf.keras.backend.clear_session()\n            if hasattr(tf.compat, 'v1'):\n                tf.compat.v1.reset_default_graph()\n        except:\n            pass\n    \n    # Remove loaded modules to free memory\n    modules_to_remove = ['xgboost', 'lightgbm', 'tensorflow', 'torch', 'sklearn', 'gplearn', 'catboost']\n    for module in list(sys.modules.keys()):\n        if any(module.startswith(mod) for mod in modules_to_remove):\n            try:\n                del sys.modules[module]\n            except:\n                pass\n    \n    # Multiple garbage collection passes\n    for _ in range(3):\n        gc.collect()\n\n# Configure compute environment\ndef configure_compute_environment():\n    \"\"\"Configure compute environment for optimal performance\"\"\"\n    os.environ['CUDA_VISIBLE_DEVICES'] = '-1'  # Default to CPU\n    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n    os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\n    os.environ['OMP_NUM_THREADS'] = '4'\n    os.environ['MKL_NUM_THREADS'] = '4'\n\n@dataclass\nclass ModelOutput:\n    \"\"\"Represents an output file from a model\"\"\"\n    file_path: str\n    file_type: str  # 'submission', 'analysis', 'features', etc.\n    created_at: datetime\n    metadata: Dict[str, Any] = field(default_factory=dict)\n\n@dataclass\nclass ModelRecord:\n    \"\"\"Complete record of a model's execution and outputs\"\"\"\n    name: str\n    directory: str\n    status: str = \"not_started\"\n    score: Optional[float] = None\n    error_message: Optional[str] = None\n    start_time: Optional[datetime] = None\n    end_time: Optional[datetime] = None\n    outputs: Dict[str, ModelOutput] = field(default_factory=dict)  # key: output_name, value: ModelOutput\n    config: Dict[str, Any] = field(default_factory=dict)\n    \n    def add_output(self, output_name: str, file_path: str, file_type: str = \"submission\", metadata: Optional[Dict] = None):\n        \"\"\"Register an output file from this model\"\"\"\n        self.outputs[output_name] = ModelOutput(\n            file_path=file_path,\n            file_type=file_type,\n            created_at=datetime.now(),\n            metadata=metadata or {}\n        )\n    \n    def get_submission_files(self) -> List[str]:\n        \"\"\"Get all submission files from this model\"\"\"\n        return [output.file_path for output in self.outputs.values() \n                if output.file_type == \"submission\" and os.path.exists(output.file_path)]\n    \n    def to_dict(self) -> Dict[str, Any]:\n        \"\"\"Convert to dictionary for serialization\"\"\"\n        return {\n            'name': self.name,\n            'directory': self.directory,\n            'status': self.status,\n            'score': self.score,\n            'error_message': self.error_message,\n            'start_time': self.start_time.isoformat() if self.start_time else None,\n            'end_time': self.end_time.isoformat() if self.end_time else None,\n            'outputs': {\n                name: {\n                    'file_path': output.file_path,\n                    'file_type': output.file_type,\n                    'created_at': output.created_at.isoformat(),\n                    'metadata': output.metadata\n                }\n                for name, output in self.outputs.items()\n            },\n            'config': self.config\n        }\n    \n    @classmethod\n    def from_dict(cls, data: Dict[str, Any]) -> 'ModelRecord':\n        \"\"\"Create from dictionary\"\"\"\n        # Convert time fields\n        if data.get('start_time'):\n            data['start_time'] = datetime.fromisoformat(data['start_time'])\n        if data.get('end_time'):\n            data['end_time'] = datetime.fromisoformat(data['end_time'])\n        \n        # Convert outputs\n        outputs = {}\n        for name, output_data in data.get('outputs', {}).items():\n            outputs[name] = ModelOutput(\n                file_path=output_data['file_path'],\n                file_type=output_data['file_type'],\n                created_at=datetime.fromisoformat(output_data['created_at']),\n                metadata=output_data.get('metadata', {})\n            )\n        data['outputs'] = outputs\n        \n        return cls(**data)\n\n@dataclass\nclass GlobalConfig:\n    \"\"\"Central configuration with dynamic model registry\"\"\"\n    # Base paths\n    base_dir: str = \"/kaggle/working/sub-models\"\n    train_path: str = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path: str = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path: str = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Model registry - using ModelRecord for complete tracking\n    model_registry: Dict[str, ModelRecord] = field(default_factory=dict)\n    \n    # Pipeline metadata\n    pipeline_start_time: Optional[datetime] = None\n    pipeline_end_time: Optional[datetime] = None\n    \n    def __post_init__(self):\n        \"\"\"Initialize configuration\"\"\"\n        Path(self.base_dir).mkdir(parents=True, exist_ok=True)\n        self.load_state()\n    \n    def register_model(self, name: str, directory: Optional[str] = None, config: Optional[Dict] = None) -> ModelRecord:\n        \"\"\"Register a new model in the pipeline\"\"\"\n        if directory is None:\n            directory = os.path.join(self.base_dir, name)\n        \n        # Create directory if it doesn't exist\n        Path(directory).mkdir(parents=True, exist_ok=True)\n        \n        # Create or update model record\n        if name in self.model_registry:\n            model_record = self.model_registry[name]\n            model_record.directory = directory\n            if config:\n                model_record.config.update(config)\n        else:\n            model_record = ModelRecord(\n                name=name,\n                directory=directory,\n                config=config or {}\n            )\n            self.model_registry[name] = model_record\n        \n        self.save_state()\n        return model_record\n    \n    def register_model_output(self, model_name: str, output_name: str, file_path: str, \n                            file_type: str = \"submission\", metadata: Optional[Dict] = None):\n        \"\"\"Register an output file from a model\"\"\"\n        if model_name not in self.model_registry:\n            self.register_model(model_name)\n        \n        model_record = self.model_registry[model_name]\n        model_record.add_output(output_name, file_path, file_type, metadata)\n        self.save_state()\n    \n    def update_model_status(self, name: str, status: str, \n                           score: Optional[float] = None,\n                           error_message: Optional[str] = None):\n        \"\"\"Update the status of a model\"\"\"\n        if name not in self.model_registry:\n            self.register_model(name)\n        \n        model = self.model_registry[name]\n        model.status = status\n        \n        if status == \"running\":\n            model.start_time = datetime.now()\n        elif status in [\"completed\", \"failed\"]:\n            model.end_time = datetime.now()\n        \n        if score is not None:\n            model.score = score\n        \n        if error_message is not None:\n            model.error_message = error_message\n        \n        self.save_state()\n    \n    def get_model_submissions(self, model_name: str) -> List[str]:\n        \"\"\"Get all submission files for a specific model\"\"\"\n        if model_name not in self.model_registry:\n            return []\n        \n        return self.model_registry[model_name].get_submission_files()\n    \n    def get_all_submissions(self) -> Dict[str, List[str]]:\n        \"\"\"Get all available submission files from all models\"\"\"\n        submissions = {}\n        for name, model in self.model_registry.items():\n            if model.status == \"completed\":\n                submission_files = model.get_submission_files()\n                if submission_files:\n                    submissions[name] = submission_files\n        return submissions\n    \n    def get_latest_submission(self, model_name: str) -> Optional[str]:\n        \"\"\"Get the most recent submission file from a model\"\"\"\n        submissions = self.get_model_submissions(model_name)\n        if not submissions:\n            return None\n        \n        # Get the output with the latest creation time\n        model = self.model_registry[model_name]\n        submission_outputs = [(name, output) for name, output in model.outputs.items() \n                            if output.file_type == \"submission\" and os.path.exists(output.file_path)]\n        \n        if not submission_outputs:\n            return None\n        \n        # Sort by creation time and return the latest\n        submission_outputs.sort(key=lambda x: x[1].created_at, reverse=True)\n        return submission_outputs[0][1].file_path\n    \n    def get_model_summary(self) -> pd.DataFrame:\n        \"\"\"Get a summary of all models as a DataFrame\"\"\"\n        data = []\n        for name, model in self.model_registry.items():\n            runtime = None\n            if model.start_time and model.end_time:\n                runtime = (model.end_time - model.start_time).total_seconds()\n            \n            submission_count = len(model.get_submission_files())\n            \n            data.append({\n                'Model': name,\n                'Status': model.status,\n                'Score': model.score,\n                'Runtime (s)': runtime,\n                'Directory': model.directory,\n                'Submissions': submission_count,\n                'Total Outputs': len(model.outputs),\n                'Error': model.error_message[:50] if model.error_message else None\n            })\n        \n        return pd.DataFrame(data)\n    \n    def get_execution_summary(self) -> str:\n        \"\"\"Get a formatted summary of model execution status\"\"\"\n        summary_df = self.get_model_summary()\n        summary_lines = []\n        \n        if summary_df.empty:\n            return \"No models registered yet.\"\n        \n        # Status counts\n        status_counts = summary_df['Status'].value_counts()\n        summary_lines.append(f\"Total models: {len(summary_df)}\")\n        \n        for status in ['completed', 'failed', 'running', 'not_started']:\n            count = status_counts.get(status, 0)\n            if count > 0:\n                emoji = {'completed': '✅', 'failed': '❌', 'running': '🔄', 'not_started': '⏸️'}[status]\n                summary_lines.append(f\"{emoji} {status}: {count}\")\n        \n        # Completed models with outputs\n        completed = summary_df[summary_df['Status'] == 'completed']\n        if not completed.empty:\n            summary_lines.append(\"\\nCompleted models:\")\n            for _, row in completed.iterrows():\n                score_str = f\"score: {row['Score']:.4f}\" if pd.notna(row['Score']) else \"score: N/A\"\n                outputs_str = f\"outputs: {row['Total Outputs']}\"\n                summary_lines.append(f\"  ✅ {row['Model']} ({score_str}, {outputs_str})\")\n        \n        # Failed models\n        failed = summary_df[summary_df['Status'] == 'failed']\n        if not failed.empty:\n            summary_lines.append(\"\\nFailed models:\")\n            for _, row in failed.iterrows():\n                error_str = row['Error'] if pd.notna(row['Error']) else \"Unknown error\"\n                summary_lines.append(f\"  ❌ {row['Model']}: {error_str}...\")\n        \n        return \"\\n\".join(summary_lines)\n    \n    def save_state(self):\n        \"\"\"Save current state to disk\"\"\"\n        state = {\n            'model_registry': {name: model.to_dict() for name, model in self.model_registry.items()},\n            'pipeline_start_time': self.pipeline_start_time.isoformat() if self.pipeline_start_time else None,\n            'pipeline_end_time': self.pipeline_end_time.isoformat() if self.pipeline_end_time else None\n        }\n        \n        state_path = os.path.join(self.base_dir, 'pipeline_state.json')\n        with open(state_path, 'w') as f:\n            json.dump(state, f, indent=2)\n        \n        # Save summary as CSV\n        summary_path = os.path.join(self.base_dir, 'model_summary.csv')\n        self.get_model_summary().to_csv(summary_path, index=False)\n    \n    def load_state(self):\n        \"\"\"Load state from disk if available\"\"\"\n        state_path = os.path.join(self.base_dir, 'pipeline_state.json')\n        \n        if os.path.exists(state_path):\n            try:\n                with open(state_path, 'r') as f:\n                    state = json.load(f)\n                \n                self.model_registry = {\n                    name: ModelRecord.from_dict(data) \n                    for name, data in state.get('model_registry', {}).items()\n                }\n                \n                if state.get('pipeline_start_time'):\n                    self.pipeline_start_time = datetime.fromisoformat(state['pipeline_start_time'])\n                if state.get('pipeline_end_time'):\n                    self.pipeline_end_time = datetime.fromisoformat(state['pipeline_end_time'])\n                    \n            except Exception as e:\n                print(f\"Warning: Could not load previous state: {e}\")\n    \n    def reset_pipeline(self):\n        \"\"\"Reset all models to initial state\"\"\"\n        self.model_registry.clear()\n        self.pipeline_start_time = None\n        self.pipeline_end_time = None\n        self.save_state()\n\n# Data utility functions\ndef reduce_memory_usage(df: pd.DataFrame, verbose: bool = True) -> pd.DataFrame:\n    \"\"\"Reduce memory usage by optimizing data types\"\"\"\n    if verbose:\n        start_mem = df.memory_usage().sum() / 1024**2\n        print(f\"Memory usage before: {start_mem:.2f} MB\")\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    if verbose:\n        end_mem = df.memory_usage().sum() / 1024**2\n        print(f\"Memory usage after: {end_mem:.2f} MB ({100 * (start_mem - end_mem) / start_mem:.1f}% reduction)\")\n    \n    return df\n\n# Initialize environment\nconfigure_compute_environment()\n\n# Initialize global configuration\nglobal_config = GlobalConfig()\n\nprint(\"🚀 DRW Crypto Prediction Pipeline - Setup Complete\")\nprint(\"=\"*80)\nprint(f\"Base directory: {global_config.base_dir}\")\nprint(f\"Training data: {global_config.train_path}\")\nprint(f\"Test data: {global_config.test_path}\")\nprint(f\"\\nModels will be automatically registered when they run.\")\nprint(\"Each model should register its outputs using global_config.register_model_output()\")\n\n# Display execution status if any\nif global_config.model_registry:\n    print(\"\\nPrevious execution status:\")\n    print(global_config.get_execution_summary())\nelse:\n    print(\"\\nNo previous execution history found - starting fresh\")\nprint(\"=\"*80)","metadata":{"trusted":true,"jupyter":{"outputs_hidden":true},"execution":{"iopub.status.busy":"2025-06-04T16:05:29.173573Z","iopub.execute_input":"2025-06-04T16:05:29.173854Z","iopub.status.idle":"2025-06-04T16:05:34.00274Z","shell.execute_reply.started":"2025-06-04T16:05:29.173836Z","shell.execute_reply":"2025-06-04T16:05:34.001952Z"},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # XGBOOST THREE-MODEL PIPELINE IMPLEMENTATION - ENHANCED FOR META-LEARNING\n# # !/usr/bin/env python\n# # -*- coding: utf-8 -*-\n# \"\"\"\n# DRW Crypto Market Prediction - XGBoost Three-Model Pipeline\n# Enhanced version that saves comprehensive OOF predictions and metadata for meta-learning\n# \"\"\"\n\n# import subprocess\n# import sys\n# import pandas as pd\n# import numpy as np\n# from scipy.stats import pearsonr\n# from typing import List, Dict, Tuple, Optional, Any\n# from pathlib import Path\n# import json\n# import gc\n# import warnings\n# import os\n# from sklearn.model_selection import KFold\n# import matplotlib.pyplot as plt\n\n# warnings.filterwarnings('ignore')\n\n# # Install required packages for this pipeline\n# print(\"Installing packages for XGBoost pipeline...\")\n# packages_to_install = [\n#     'xgboost==2.0.3',\n#     'shap==0.44.0'\n# ]\n\n# for package in packages_to_install:\n#     subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n\n# # Import XGBoost after installation\n# import xgboost as xgb\n\n# # Custom JSON encoder to handle numpy types\n# class NumpyEncoder(json.JSONEncoder):\n#     \"\"\"Custom encoder for numpy data types\"\"\"\n#     def default(self, obj):\n#         if isinstance(obj, (np.integer, np.int_, np.intc, np.intp, np.int8,\n#                           np.int16, np.int32, np.int64, np.uint8, np.uint16,\n#                           np.uint32, np.uint64)):\n#             return int(obj)\n#         elif isinstance(obj, (np.floating, np.float_, np.float16, np.float32, np.float64)):\n#             return float(obj)\n#         elif isinstance(obj, (np.complex_, np.complex64, np.complex128)):\n#             return {'real': obj.real, 'imag': obj.imag}\n#         elif isinstance(obj, np.ndarray):\n#             return obj.tolist()\n#         elif isinstance(obj, np.bool_):\n#             return bool(obj)\n#         elif isinstance(obj, np.void):\n#             return None\n#         return super(NumpyEncoder, self).default(obj)\n\n# class XGBoostConfiguration:\n#     \"\"\"Configuration for XGBoost Three-Model Pipeline\"\"\"\n    \n#     def __init__(self):\n#         # Model registration\n#         self.model_name = \"xgboost\"\n#         self.model_directory = os.path.join(global_config.base_dir, \"triple_xgboost\")\n        \n#         # Register model with global configuration\n#         global_config.register_model(self.model_name, self.model_directory)\n        \n#         # Data paths from global configuration\n#         self.train_path = global_config.train_path\n#         self.test_path = global_config.test_path\n#         self.sample_sub_path = global_config.sample_sub_path\n        \n#         # Model-specific feature selection\n#         self.selected_features = [\n#             \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n#             \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n#             \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n#         ]\n        \n#         # XGBoost hyperparameters\n#         self.xgb_params = {\n#             \"tree_method\": \"hist\",\n#             \"device\": \"cpu\",\n#             \"colsample_bylevel\": 0.4778015829774066,\n#             \"colsample_bynode\": 0.362764358742407,\n#             \"colsample_bytree\": 0.7107423488010493,\n#             \"gamma\": 1.7094857725240398,\n#             \"learning_rate\": 0.02213323588455387,\n#             \"max_depth\": 20,\n#             \"max_leaves\": 12,\n#             \"min_child_weight\": 16,\n#             \"n_estimators\": 1667,\n#             \"n_jobs\": -1,\n#             \"random_state\": 42,\n#             \"reg_alpha\": 39.352415706891264,\n#             \"reg_lambda\": 75.44843704068275,\n#             \"subsample\": 0.06566669853471274,\n#             \"verbosity\": 0,\n#             \"eval_metric\": \"rmse\"  # Add explicit eval metric\n#         }\n        \n#         # Model configurations for time windows\n#         self.model_configs = [\n#             {\"name\": \"model_1_full_data\", \"percent\": 1.00, \"description\": \"Full Data\"},\n#             {\"name\": \"model_2_recent_75\", \"percent\": 0.75, \"description\": \"75% Recent\"},\n#             {\"name\": \"model_3_recent_50\", \"percent\": 0.50, \"description\": \"50% Recent\"}\n#         ]\n        \n#         # Cross-validation parameters\n#         self.n_folds = 5\n#         self.random_state = 42\n#         self.shuffle = True\n#         self.decay_factor = 0.95\n#         self.early_stopping_rounds = 25\n#         self.verbose_eval = 200\n        \n#         # Output paths\n#         self.intermediate_dir = os.path.join(self.model_directory, \"sub_models\")\n#         self.submission_file = os.path.join(self.model_directory, \"submission.csv\")\n#         self.results_file = os.path.join(self.model_directory, \"ensemble_results.csv\")\n#         self.shap_features_path = os.path.join(self.model_directory, \"shap_features.csv\")\n        \n#         # New paths for meta-learning data\n#         self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n#         self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n#         self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n#         self.row_level_metrics_path = os.path.join(self.model_directory, \"row_level_metrics.csv\")\n        \n#         # Ensure directories exist\n#         Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n#         Path(self.intermediate_dir).mkdir(parents=True, exist_ok=True)\n\n# class XGBoostDataProcessor:\n#     \"\"\"Data processing utilities for XGBoost pipeline\"\"\"\n    \n#     @staticmethod\n#     def create_time_weights(n_samples: int, decay_factor: float = 0.95) -> np.ndarray:\n#         \"\"\"Create exponential decay weights for time series data\"\"\"\n#         positions = np.arange(n_samples)\n#         normalized_positions = positions / (n_samples - 1)\n#         weights = decay_factor ** (1 - normalized_positions)\n#         weights = weights * n_samples / weights.sum()\n#         return weights\n    \n#     @staticmethod\n#     def load_data(config: XGBoostConfiguration) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n#         \"\"\"Load and prepare data for XGBoost models\"\"\"\n#         print(\"Loading data...\")\n#         train = pd.read_parquet(config.train_path).reset_index(drop=True)\n#         test = pd.read_parquet(config.test_path).reset_index(drop=True)\n#         sample = pd.read_csv(config.sample_sub_path)\n        \n#         # Add row indices for tracking\n#         train['original_index'] = range(len(train))\n#         test['original_index'] = range(len(test))\n        \n#         # Verify feature availability\n#         available_features = [f for f in config.selected_features if f in train.columns]\n#         missing_features = set(config.selected_features) - set(available_features)\n        \n#         if missing_features:\n#             print(f\"Warning: {len(missing_features)} features not found in data\")\n#             print(f\"Missing features: {missing_features}\")\n        \n#         # Select only available features (keeping original_index)\n#         train = train[available_features + [\"label\", \"original_index\"]]\n#         test = test[available_features + [\"original_index\"]]\n        \n#         # Reduce memory usage\n#         train = reduce_memory_usage(train, verbose=False)\n#         test = reduce_memory_usage(test, verbose=False)\n        \n#         print(f\"Data loaded - Train: {train.shape}, Test: {test.shape}\")\n#         print(f\"Using {len(available_features)} features\")\n        \n#         return train, test, sample\n\n# class XGBoostModelTrainer:\n#     \"\"\"Model training utilities for XGBoost pipeline\"\"\"\n    \n#     def __init__(self, config: XGBoostConfiguration):\n#         self.config = config\n    \n#     def train_model(self, X_train: pd.DataFrame, y_train: pd.Series, \n#                    X_valid: pd.DataFrame, y_valid: pd.Series, \n#                    sample_weights: np.ndarray) -> Tuple[xgb.XGBRegressor, Dict[str, Any]]:\n#         \"\"\"Train a single XGBoost model with early stopping and return metadata\"\"\"\n#         model = xgb.XGBRegressor(**self.config.xgb_params)\n        \n#         # Train the model\n#         model.fit(\n#             X_train, y_train,\n#             sample_weight=sample_weights,\n#             eval_set=[(X_valid, y_valid)],\n#             early_stopping_rounds=self.config.early_stopping_rounds,\n#             verbose=self.config.verbose_eval\n#         )\n        \n#         # Extract training metadata - convert numpy types to Python types\n#         metadata = {\n#             'best_iteration': int(model.best_iteration) if hasattr(model, 'best_iteration') else int(model.n_estimators),\n#             'best_score': float(model.best_score) if hasattr(model, 'best_score') else None,\n#             'n_features_used': int(len(X_train.columns)),\n#             'train_size': int(len(X_train)),\n#             'valid_size': int(len(X_valid)),\n#             'feature_importance': {k: float(v) for k, v in zip(X_train.columns, model.feature_importances_)}\n#         }\n        \n#         # Try to get evaluation results if available\n#         if hasattr(model, 'evals_result_'):\n#             metadata['eval_results'] = model.evals_result_\n        \n#         return model, metadata\n    \n#     def prepare_windowed_data(self, train_df: pd.DataFrame, train_idx: np.ndarray,\n#                             cutoff: int, features: List[str]) -> Tuple[pd.DataFrame, pd.Series, np.ndarray, np.ndarray]:\n#         \"\"\"Prepare training data for windowed models\"\"\"\n#         train_idx_recent = train_idx[train_idx >= cutoff]\n#         train_idx_recent_adjusted = train_idx_recent - cutoff\n        \n#         train_recent = train_df.iloc[cutoff:].reset_index(drop=True)\n        \n#         X_train = train_recent.iloc[train_idx_recent_adjusted][features]\n#         y_train = train_recent.iloc[train_idx_recent_adjusted][\"label\"]\n#         original_indices = train_recent.iloc[train_idx_recent_adjusted][\"original_index\"].values\n        \n#         sample_weights_recent = XGBoostDataProcessor.create_time_weights(\n#             len(train_recent), self.config.decay_factor\n#         )\n#         train_weights = sample_weights_recent[train_idx_recent_adjusted]\n        \n#         return X_train, y_train, train_weights, original_indices\n\n# class XGBoostEnsembleBuilder:\n#     \"\"\"Ensemble building and evaluation utilities with meta-learning features\"\"\"\n    \n#     def __init__(self, config: XGBoostConfiguration):\n#         self.config = config\n    \n#     def evaluate_models(self, predictions: Dict[str, np.ndarray], \n#                        train_labels: pd.Series) -> pd.DataFrame:\n#         \"\"\"Evaluate individual models and ensemble combinations\"\"\"\n#         scores = {}\n        \n#         # Individual model scores\n#         for model_name, preds in predictions.items():\n#             scores[model_name] = pearsonr(train_labels, preds)[0]\n        \n#         # Simple ensemble\n#         simple_ensemble = np.mean(list(predictions.values()), axis=0)\n#         scores['simple_ensemble'] = pearsonr(train_labels, simple_ensemble)[0]\n        \n#         # Weighted ensemble based on individual scores\n#         weights = np.array([scores[name] for name in predictions.keys()])\n#         weights = weights / weights.sum()\n        \n#         weighted_ensemble = np.average(list(predictions.values()), axis=0, weights=weights)\n#         scores['weighted_ensemble'] = pearsonr(train_labels, weighted_ensemble)[0]\n        \n#         # Create results dataframe\n#         results_data = []\n#         for i, (name, score) in enumerate(scores.items()):\n#             weight = weights[i] if i < len(weights) else np.nan\n#             results_data.append({\n#                 'model': name,\n#                 'pearson_correlation': score,\n#                 'weight_in_final': weight\n#             })\n        \n#         return pd.DataFrame(results_data)\n    \n#     def create_meta_features(self, predictions: Dict[str, np.ndarray], \n#                            train_labels: np.ndarray) -> pd.DataFrame:\n#         \"\"\"Create comprehensive meta-features for meta-learning\"\"\"\n#         meta_features = {}\n        \n#         # Basic predictions from each model\n#         for model_name, preds in predictions.items():\n#             meta_features[f'pred_{model_name}'] = preds\n        \n#         # Model agreement features\n#         pred_array = np.column_stack(list(predictions.values()))\n#         meta_features['pred_mean'] = np.mean(pred_array, axis=1)\n#         meta_features['pred_std'] = np.std(pred_array, axis=1)\n#         meta_features['pred_min'] = np.min(pred_array, axis=1)\n#         meta_features['pred_max'] = np.max(pred_array, axis=1)\n#         meta_features['pred_range'] = meta_features['pred_max'] - meta_features['pred_min']\n#         meta_features['pred_cv'] = meta_features['pred_std'] / (np.abs(meta_features['pred_mean']) + 1e-8)\n        \n#         # Model disagreement indicators\n#         meta_features['model_agreement'] = 1 / (meta_features['pred_std'] + 1e-8)\n        \n#         # Distance from consensus\n#         for model_name, preds in predictions.items():\n#             meta_features[f'dist_from_mean_{model_name}'] = np.abs(preds - meta_features['pred_mean'])\n        \n#         # Pairwise differences\n#         model_names = list(predictions.keys())\n#         for i in range(len(model_names)):\n#             for j in range(i + 1, len(model_names)):\n#                 diff = predictions[model_names[i]] - predictions[model_names[j]]\n#                 meta_features[f'diff_{model_names[i]}_{model_names[j]}'] = diff\n#                 meta_features[f'abs_diff_{model_names[i]}_{model_names[j]}'] = np.abs(diff)\n        \n#         # Rank of each prediction\n#         for model_name, preds in predictions.items():\n#             rank = np.argsort(np.argsort(preds))\n#             meta_features[f'rank_{model_name}'] = rank / len(preds)\n        \n#         # Error features (for training data)\n#         for model_name, preds in predictions.items():\n#             meta_features[f'error_{model_name}'] = preds - train_labels\n#             meta_features[f'abs_error_{model_name}'] = np.abs(preds - train_labels)\n#             meta_features[f'squared_error_{model_name}'] = (preds - train_labels) ** 2\n        \n#         # Best and worst model per row\n#         errors = np.abs(pred_array - train_labels.reshape(-1, 1))\n#         meta_features['best_model_idx'] = np.argmin(errors, axis=1)\n#         meta_features['worst_model_idx'] = np.argmax(errors, axis=1)\n#         meta_features['best_model_error'] = np.min(errors, axis=1)\n#         meta_features['worst_model_error'] = np.max(errors, axis=1)\n#         meta_features['error_range'] = meta_features['worst_model_error'] - meta_features['best_model_error']\n        \n#         # Model selection features\n#         for i, model_name in enumerate(model_names):\n#             meta_features[f'is_best_{model_name}'] = (meta_features['best_model_idx'] == i).astype(int)\n#             meta_features[f'is_worst_{model_name}'] = (meta_features['worst_model_idx'] == i).astype(int)\n        \n#         return pd.DataFrame(meta_features)\n    \n#     def calculate_model_performance_segments(self, predictions: Dict[str, np.ndarray],\n#                                            train_labels: np.ndarray) -> Dict[str, Any]:\n#         \"\"\"Calculate model performance across different data segments\"\"\"\n#         performance_segments = {}\n        \n#         # Overall performance\n#         performance_segments['overall'] = {}\n#         for model_name, preds in predictions.items():\n#             performance_segments['overall'][model_name] = {\n#                 'correlation': float(pearsonr(preds, train_labels)[0]),\n#                 'mae': float(np.mean(np.abs(preds - train_labels))),\n#                 'rmse': float(np.sqrt(np.mean((preds - train_labels) ** 2)))\n#             }\n        \n#         # Performance by prediction quantiles\n#         pred_mean = np.mean(list(predictions.values()), axis=0)\n#         quantiles = np.percentile(pred_mean, [25, 50, 75])\n        \n#         performance_segments['by_prediction_level'] = {}\n#         for q_idx, (low, high) in enumerate([(float('-inf'), quantiles[0]), \n#                                             (quantiles[0], quantiles[1]),\n#                                             (quantiles[1], quantiles[2]),\n#                                             (quantiles[2], float('inf'))]):\n#             mask = (pred_mean >= low) & (pred_mean < high)\n#             if np.sum(mask) > 10:\n#                 performance_segments['by_prediction_level'][f'q{q_idx+1}'] = {}\n#                 for model_name, preds in predictions.items():\n#                     performance_segments['by_prediction_level'][f'q{q_idx+1}'][model_name] = {\n#                         'correlation': float(pearsonr(preds[mask], train_labels[mask])[0]) if np.sum(mask) > 1 else 0,\n#                         'mae': float(np.mean(np.abs(preds[mask] - train_labels[mask]))),\n#                         'n_samples': int(np.sum(mask))\n#                     }\n        \n#         # Performance by time (first half vs second half)\n#         mid_point = len(train_labels) // 2\n#         performance_segments['by_time'] = {\n#             'first_half': {},\n#             'second_half': {}\n#         }\n        \n#         for model_name, preds in predictions.items():\n#             performance_segments['by_time']['first_half'][model_name] = {\n#                 'correlation': float(pearsonr(preds[:mid_point], train_labels[:mid_point])[0]),\n#                 'mae': float(np.mean(np.abs(preds[:mid_point] - train_labels[:mid_point])))\n#             }\n#             performance_segments['by_time']['second_half'][model_name] = {\n#                 'correlation': float(pearsonr(preds[mid_point:], train_labels[mid_point:])[0]),\n#                 'mae': float(np.mean(np.abs(preds[mid_point:] - train_labels[mid_point:])))\n#             }\n        \n#         # Performance by volatility (using prediction std as proxy)\n#         pred_std = np.std(list(predictions.values()), axis=0)\n#         high_vol_mask = pred_std > np.median(pred_std)\n#         low_vol_mask = ~high_vol_mask\n        \n#         performance_segments['by_volatility'] = {\n#             'high_volatility': {},\n#             'low_volatility': {}\n#         }\n        \n#         for model_name, preds in predictions.items():\n#             if np.sum(high_vol_mask) > 10:\n#                 performance_segments['by_volatility']['high_volatility'][model_name] = {\n#                     'correlation': float(pearsonr(preds[high_vol_mask], train_labels[high_vol_mask])[0]),\n#                     'mae': float(np.mean(np.abs(preds[high_vol_mask] - train_labels[high_vol_mask]))),\n#                     'n_samples': int(np.sum(high_vol_mask))\n#                 }\n#             if np.sum(low_vol_mask) > 10:\n#                 performance_segments['by_volatility']['low_volatility'][model_name] = {\n#                     'correlation': float(pearsonr(preds[low_vol_mask], train_labels[low_vol_mask])[0]),\n#                     'mae': float(np.mean(np.abs(preds[low_vol_mask] - train_labels[low_vol_mask]))),\n#                     'n_samples': int(np.sum(low_vol_mask))\n#                 }\n        \n#         return performance_segments\n    \n#     def create_final_ensemble(self, test_predictions: Dict[str, np.ndarray],\n#                             oof_scores: Dict[str, float]) -> np.ndarray:\n#         \"\"\"Create final ensemble predictions\"\"\"\n#         weights = np.array([oof_scores[name] for name in test_predictions.keys()])\n#         weights = weights / weights.sum()\n        \n#         final_predictions = np.average(list(test_predictions.values()), axis=0, weights=weights)\n        \n#         return final_predictions\n\n# class XGBoostThreeModelPipeline:\n#     \"\"\"Main pipeline orchestrating the three-model XGBoost approach with meta-learning enhancements\"\"\"\n    \n#     def __init__(self):\n#         self.config = XGBoostConfiguration()\n#         self.data_processor = XGBoostDataProcessor()\n#         self.trainer = XGBoostModelTrainer(self.config)\n#         self.ensemble_builder = XGBoostEnsembleBuilder(self.config)\n    \n#     def run_cross_validation(self, train_df: pd.DataFrame, test_df: pd.DataFrame) -> Dict[str, Any]:\n#         \"\"\"Execute cross-validation for all three models with comprehensive tracking\"\"\"\n#         features = [c for c in train_df.columns if c not in [\"label\", \"original_index\"]]\n        \n#         # Calculate cutoffs for time windows\n#         cutoff_75 = int(len(train_df) * 0.25)\n#         cutoff_50 = int(len(train_df) * 0.50)\n        \n#         print(f\"\\nTime window configurations:\")\n#         print(f\"  Model 1: Full data ({len(train_df):,} samples)\")\n#         print(f\"  Model 2: 75% recent ({len(train_df) - cutoff_75:,} samples)\")\n#         print(f\"  Model 3: 50% recent ({len(train_df) - cutoff_50:,} samples)\")\n        \n#         # Initialize prediction storage\n#         oof_predictions = {\n#             'model_1': np.full(len(train_df), np.nan),\n#             'model_2': np.full(len(train_df), np.nan),\n#             'model_3': np.full(len(train_df), np.nan)\n#         }\n        \n#         test_predictions = {\n#             'model_1': np.zeros(len(test_df)),\n#             'model_2': np.zeros(len(test_df)),\n#             'model_3': np.zeros(len(test_df))\n#         }\n        \n#         # Store fold assignments\n#         fold_assignments = np.full(len(train_df), -1)\n        \n#         # Store model metadata\n#         model_metadata = {\n#             'model_1': [],\n#             'model_2': [],\n#             'model_3': []\n#         }\n        \n#         # Create sample weights for full dataset\n#         sample_weights_full = self.data_processor.create_time_weights(\n#             len(train_df), self.config.decay_factor\n#         )\n        \n#         # Cross-validation\n#         kf = KFold(n_splits=self.config.n_folds, shuffle=self.config.shuffle, \n#                   random_state=self.config.random_state)\n        \n#         for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df)):\n#             print(f\"\\nProcessing fold {fold + 1}/{self.config.n_folds}\")\n            \n#             # Store fold assignments\n#             fold_assignments[valid_idx] = fold\n            \n#             X_valid = train_df.iloc[valid_idx][features]\n#             y_valid = train_df.iloc[valid_idx][\"label\"]\n#             X_test = test_df[features]\n            \n#             # Model 1: Full data\n#             X_train = train_df.iloc[train_idx][features]\n#             y_train = train_df.iloc[train_idx][\"label\"]\n#             train_weights = sample_weights_full[train_idx]\n            \n#             model1, metadata1 = self.trainer.train_model(X_train, y_train, X_valid, y_valid, train_weights)\n#             oof_predictions['model_1'][valid_idx] = model1.predict(X_valid)\n#             test_predictions['model_1'] += model1.predict(X_test) / self.config.n_folds\n            \n#             metadata1['fold'] = fold\n#             model_metadata['model_1'].append(metadata1)\n            \n#             # Model 2: 75% recent\n#             X_train_75, y_train_75, weights_75, _ = self.trainer.prepare_windowed_data(\n#                 train_df, train_idx, cutoff_75, features\n#             )\n            \n#             if len(X_train_75) > 0:\n#                 model2, metadata2 = self.trainer.train_model(X_train_75, y_train_75, X_valid, y_valid, weights_75)\n                \n#                 # Handle predictions for validation set\n#                 valid_idx_recent = valid_idx[valid_idx >= cutoff_75]\n#                 if len(valid_idx_recent) > 0:\n#                     X_valid_recent = train_df.iloc[valid_idx_recent][features]\n#                     oof_predictions['model_2'][valid_idx_recent] = model2.predict(X_valid_recent)\n                \n#                 # Use model 1 predictions for samples before cutoff\n#                 valid_idx_old = valid_idx[valid_idx < cutoff_75]\n#                 if len(valid_idx_old) > 0:\n#                     oof_predictions['model_2'][valid_idx_old] = oof_predictions['model_1'][valid_idx_old]\n                \n#                 test_predictions['model_2'] += model2.predict(X_test) / self.config.n_folds\n                \n#                 metadata2['fold'] = fold\n#                 metadata2['cutoff'] = cutoff_75\n#                 model_metadata['model_2'].append(metadata2)\n            \n#             # Model 3: 50% recent\n#             X_train_50, y_train_50, weights_50, _ = self.trainer.prepare_windowed_data(\n#                 train_df, train_idx, cutoff_50, features\n#             )\n            \n#             if len(X_train_50) > 0:\n#                 model3, metadata3 = self.trainer.train_model(X_train_50, y_train_50, X_valid, y_valid, weights_50)\n                \n#                 valid_idx_recent = valid_idx[valid_idx >= cutoff_50]\n#                 if len(valid_idx_recent) > 0:\n#                     X_valid_recent = train_df.iloc[valid_idx_recent][features]\n#                     oof_predictions['model_3'][valid_idx_recent] = model3.predict(X_valid_recent)\n                \n#                 valid_idx_old = valid_idx[valid_idx < cutoff_50]\n#                 if len(valid_idx_old) > 0:\n#                     oof_predictions['model_3'][valid_idx_old] = oof_predictions['model_1'][valid_idx_old]\n                \n#                 test_predictions['model_3'] += model3.predict(X_test) / self.config.n_folds\n                \n#                 metadata3['fold'] = fold\n#                 metadata3['cutoff'] = cutoff_50\n#                 model_metadata['model_3'].append(metadata3)\n        \n#         return {\n#             'oof_predictions': oof_predictions,\n#             'test_predictions': test_predictions,\n#             'train_labels': train_df[\"label\"],\n#             'train_indices': train_df[\"original_index\"],\n#             'fold_assignments': fold_assignments,\n#             'model_metadata': model_metadata\n#         }\n    \n#     def save_oof_predictions_and_meta_features(self, cv_results: Dict[str, Any]):\n#         \"\"\"Save comprehensive OOF predictions and meta-features for meta-learning\"\"\"\n#         # Create OOF predictions DataFrame\n#         oof_df = pd.DataFrame({\n#             'row_id': cv_results['train_indices'].values,\n#             'true_label': cv_results['train_labels'].values,\n#             'fold': cv_results['fold_assignments']\n#         })\n        \n#         # Add predictions from each model\n#         for model_name, predictions in cv_results['oof_predictions'].items():\n#             oof_df[f'pred_{model_name}'] = predictions\n        \n#         # Save OOF predictions\n#         oof_df.to_csv(self.config.oof_predictions_path, index=False)\n#         print(f\"OOF predictions saved to {self.config.oof_predictions_path}\")\n        \n#         # Register OOF predictions with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'oof_predictions',\n#             self.config.oof_predictions_path,\n#             'oof_predictions',\n#             metadata={\n#                 'n_models': len(cv_results['oof_predictions']),\n#                 'n_samples': len(oof_df),\n#                 'models': list(cv_results['oof_predictions'].keys())\n#             }\n#         )\n        \n#         # Create and save meta-features\n#         meta_features_df = self.ensemble_builder.create_meta_features(\n#             cv_results['oof_predictions'],\n#             cv_results['train_labels'].values\n#         )\n#         meta_features_df['row_id'] = cv_results['train_indices'].values\n        \n#         # Reorder columns to have row_id first\n#         cols = ['row_id'] + [col for col in meta_features_df.columns if col != 'row_id']\n#         meta_features_df = meta_features_df[cols]\n        \n#         meta_features_df.to_csv(self.config.meta_features_path, index=False)\n#         print(f\"Meta-features saved to {self.config.meta_features_path}\")\n        \n#         # Register meta-features with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'meta_features',\n#             self.config.meta_features_path,\n#             'features',\n#             metadata={\n#                 'n_features': len(meta_features_df.columns) - 1,  # Exclude row_id\n#                 'feature_names': [col for col in meta_features_df.columns if col != 'row_id']\n#             }\n#         )\n        \n#         # Calculate and save row-level metrics\n#         row_metrics = pd.DataFrame({\n#             'row_id': cv_results['train_indices'].values,\n#             'true_label': cv_results['train_labels'].values\n#         })\n        \n#         # Add error metrics for each model\n#         for model_name, predictions in cv_results['oof_predictions'].items():\n#             row_metrics[f'error_{model_name}'] = predictions - cv_results['train_labels'].values\n#             row_metrics[f'abs_error_{model_name}'] = np.abs(predictions - cv_results['train_labels'].values)\n#             row_metrics[f'squared_error_{model_name}'] = (predictions - cv_results['train_labels'].values) ** 2\n        \n#         # Add best model indicator\n#         errors = np.column_stack([row_metrics[f'abs_error_{model_name}'].values \n#                                  for model_name in cv_results['oof_predictions'].keys()])\n#         row_metrics['best_model'] = np.argmin(errors, axis=1)\n#         row_metrics['best_model_name'] = [list(cv_results['oof_predictions'].keys())[idx] \n#                                          for idx in row_metrics['best_model']]\n        \n#         row_metrics.to_csv(self.config.row_level_metrics_path, index=False)\n#         print(f\"Row-level metrics saved to {self.config.row_level_metrics_path}\")\n        \n#         # Register row-level metrics\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'row_level_metrics',\n#             self.config.row_level_metrics_path,\n#             'analysis',\n#             metadata={\n#                 'metrics': [col for col in row_metrics.columns if col not in ['row_id', 'true_label']]\n#             }\n#         )\n    \n#     def save_model_performance_analysis(self, cv_results: Dict[str, Any]):\n#         \"\"\"Save comprehensive model performance analysis\"\"\"\n#         # Calculate performance across different segments\n#         performance_segments = self.ensemble_builder.calculate_model_performance_segments(\n#             cv_results['oof_predictions'],\n#             cv_results['train_labels'].values\n#         )\n        \n#         # Add model metadata\n#         performance_data = {\n#             'model_configurations': self.config.model_configs,\n#             'model_metadata': cv_results['model_metadata'],\n#             'performance_segments': performance_segments,\n#             'hyperparameters': self.config.xgb_params,\n#             'cv_settings': {\n#                 'n_folds': self.config.n_folds,\n#                 'shuffle': self.config.shuffle,\n#                 'random_state': self.config.random_state\n#             }\n#         }\n        \n#         # Save to JSON with custom encoder\n#         with open(self.config.model_performance_path, 'w') as f:\n#             json.dump(performance_data, f, indent=2, cls=NumpyEncoder)\n        \n#         print(f\"Model performance analysis saved to {self.config.model_performance_path}\")\n        \n#         # Register performance analysis\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'model_performance',\n#             self.config.model_performance_path,\n#             'analysis',\n#             metadata={\n#                 'segments_analyzed': list(performance_segments.keys())\n#             }\n#         )\n    \n#     def run(self) -> float:\n#         \"\"\"Execute the complete XGBoost three-model pipeline with meta-learning enhancements\"\"\"\n#         print(\"\\nStarting XGBoost Three-Model Pipeline (Enhanced for Meta-Learning)\")\n#         print(\"=\"*80)\n        \n#         # Update model status\n#         global_config.update_model_status(self.config.model_name, \"running\")\n        \n#         # Load data\n#         train_df, test_df, sample_df = self.data_processor.load_data(self.config)\n        \n#         # Run cross-validation\n#         print(\"\\nRunning cross-validation...\")\n#         cv_results = self.run_cross_validation(train_df, test_df)\n        \n#         # Save OOF predictions and meta-features\n#         print(\"\\nSaving OOF predictions and meta-features...\")\n#         self.save_oof_predictions_and_meta_features(cv_results)\n        \n#         # Save model performance analysis\n#         print(\"\\nSaving model performance analysis...\")\n#         self.save_model_performance_analysis(cv_results)\n        \n#         # Evaluate models\n#         print(\"\\nEvaluating model performance...\")\n#         ensemble_results = self.ensemble_builder.evaluate_models(\n#             cv_results['oof_predictions'],\n#             cv_results['train_labels']\n#         )\n        \n#         # Display results\n#         print(\"\\nModel Performance Summary:\")\n#         for _, row in ensemble_results.iterrows():\n#             if pd.notna(row['pearson_correlation']):\n#                 print(f\"  {row['model']}: {row['pearson_correlation']:.4f}\")\n        \n#         # Get best ensemble score\n#         best_score = ensemble_results['pearson_correlation'].max()\n#         best_model = ensemble_results.loc[\n#             ensemble_results['pearson_correlation'].idxmax(), 'model'\n#         ]\n        \n#         print(f\"\\nBest performing approach: {best_model} (score: {best_score:.4f})\")\n        \n#         # Create final predictions\n#         oof_scores = {\n#             name: ensemble_results[ensemble_results['model'] == name]['pearson_correlation'].values[0]\n#             for name in cv_results['test_predictions'].keys()\n#         }\n        \n#         final_predictions = self.ensemble_builder.create_final_ensemble(\n#             cv_results['test_predictions'],\n#             oof_scores\n#         )\n        \n#         # Save submission\n#         sample_df[\"prediction\"] = final_predictions\n#         sample_df.to_csv(self.config.submission_file, index=False)\n#         print(f\"\\nSubmission saved to {self.config.submission_file}\")\n        \n#         # Register submission with enhanced metadata\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'submission',\n#             self.config.submission_file,\n#             'submission',\n#             metadata={\n#                 'best_score': float(best_score),\n#                 'best_model': best_model,\n#                 'n_models': len(self.config.model_configs),\n#                 'oof_scores': {k: float(v) for k, v in oof_scores.items()},\n#                 'has_meta_features': True\n#             }\n#         )\n        \n#         # Save ensemble results\n#         ensemble_results.to_csv(self.config.results_file, index=False)\n        \n#         # Register ensemble results\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'ensemble_results',\n#             self.config.results_file,\n#             'analysis',\n#             metadata={\n#                 'model_scores': {k: float(v) for k, v in ensemble_results.set_index('model')['pearson_correlation'].to_dict().items()}\n#             }\n#         )\n        \n#         # Generate feature importance\n#         self.save_feature_importance(train_df.drop(['original_index'], axis=1), \n#                                    test_df.drop(['original_index'], axis=1))\n        \n#         print(\"\\nXGBoost pipeline completed successfully\")\n#         print(\"Meta-learning data saved:\")\n#         print(f\"  - OOF predictions: {self.config.oof_predictions_path}\")\n#         print(f\"  - Meta-features: {self.config.meta_features_path}\")\n#         print(f\"  - Row-level metrics: {self.config.row_level_metrics_path}\")\n#         print(f\"  - Model performance: {self.config.model_performance_path}\")\n        \n#         return best_score\n    \n#     def save_feature_importance(self, train_df: pd.DataFrame, test_df: pd.DataFrame):\n#         \"\"\"Generate and save SHAP feature importance analysis\"\"\"\n#         try:\n#             import shap\n            \n#             features = [c for c in train_df.columns if c != \"label\"]\n#             sample_weights = self.data_processor.create_time_weights(\n#                 len(train_df), self.config.decay_factor\n#             )\n            \n#             print(\"\\nGenerating SHAP feature importance...\")\n            \n#             # Train model for SHAP analysis\n#             model = xgb.XGBRegressor(**self.config.xgb_params)\n#             model.fit(\n#                 train_df[features], \n#                 train_df[\"label\"],\n#                 sample_weight=sample_weights,\n#                 verbose=0\n#             )\n            \n#             # Calculate SHAP values on a sample\n#             sample_size = min(1000, len(test_df))\n#             test_sample = test_df[features].iloc[:sample_size]\n            \n#             explainer = shap.TreeExplainer(model)\n#             shap_values = explainer.shap_values(test_sample)\n            \n#             # Create feature importance dataframe\n#             feature_importance = pd.DataFrame({\n#                 'feature': features,\n#                 'importance': np.abs(shap_values).mean(axis=0)\n#             }).sort_values('importance', ascending=False)\n            \n#             feature_importance.to_csv(self.config.shap_features_path, index=False)\n#             print(f\"Feature importance saved to {self.config.shap_features_path}\")\n            \n#             # Register SHAP features output\n#             global_config.register_model_output(\n#                 self.config.model_name,\n#                 'shap_features',\n#                 self.config.shap_features_path,\n#                 'analysis',\n#                 metadata={'top_features': feature_importance.head(10)['feature'].tolist()}\n#             )\n            \n#             # Save top features\n#             print(\"\\nTop 10 most important features:\")\n#             for idx, row in feature_importance.head(10).iterrows():\n#                 print(f\"  {row['feature']}: {row['importance']:.4f}\")\n            \n#         except Exception as e:\n#             print(f\"Warning: Could not generate SHAP feature importance: {e}\")\n\n# # Main execution\n# if __name__ == \"__main__\":\n#     try:\n#         print(\"\\n📊 Running XGBoost Three-Model Pipeline (Enhanced)\")\n#         print(\"-\"*80)\n        \n#         # Clean memory before starting\n#         aggressive_memory_cleanup()\n        \n#         # Create and run pipeline\n#         pipeline = XGBoostThreeModelPipeline()\n#         final_score = pipeline.run()\n        \n#         # Update global configuration with success\n#         global_config.update_model_status('xgboost', 'completed', score=final_score)\n#         print(\"\\n✅ XGBoost pipeline completed successfully\")\n        \n#     except Exception as e:\n#         # Update global configuration with failure\n#         error_msg = str(e)\n#         global_config.update_model_status('xgboost', 'failed', error_message=error_msg)\n#         print(f\"\\n❌ XGBoost pipeline failed: {error_msg}\")\n#         raise\n        \n#     finally:\n#         # Clean up memory\n#         aggressive_memory_cleanup()\n        \n#         # Display current execution status\n#         print(\"\\n\" + \"=\"*80)\n#         print(\"Current Execution Status:\")\n#         print(global_config.get_execution_summary())\n#         print(\"=\"*80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T16:05:34.003635Z","iopub.execute_input":"2025-06-04T16:05:34.004006Z","iopub.status.idle":"2025-06-04T16:09:38.791145Z","shell.execute_reply.started":"2025-06-04T16:05:34.003986Z","shell.execute_reply":"2025-06-04T16:09:38.79026Z"},"jupyter":{"outputs_hidden":true},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# FAST MEMORY-OPTIMIZED XGBOOST WIDE FEATURES WITH PCA PIPELINE\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - Fast Memory-Optimized XGBoost Wide Features with PCA Pipeline\nOptimized for speed: ~30-60 minutes instead of 10 hours\n\"\"\"\n\nimport subprocess\nimport sys\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr\nfrom typing import List, Dict, Tuple, Optional, Any, Iterator\nfrom pathlib import Path\nimport json\nimport gc\nimport warnings\nimport os\nimport pickle\nimport h5py\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_selection import VarianceThreshold\nimport joblib\nimport pyarrow.parquet as pq\nimport pyarrow as pa\nfrom concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\nimport multiprocessing as mp\n\nwarnings.filterwarnings('ignore')\n\n# Install required packages\nprint(\"Installing packages for Fast XGBoost Pipeline...\")\npackages_to_install = [\n    'xgboost==2.0.3',\n    'scikit-learn>=1.0.0',\n    'h5py>=3.0.0',\n    'joblib>=1.0.0',\n    'pyarrow>=10.0.0'\n]\n\nfor package in packages_to_install:\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n\nimport xgboost as xgb\n\ndef convert_numpy_to_python(obj):\n    \"\"\"Convert numpy types to native Python types for JSON serialization\"\"\"\n    if isinstance(obj, np.integer):\n        return int(obj)\n    elif isinstance(obj, np.floating):\n        return float(obj)\n    elif isinstance(obj, np.ndarray):\n        return obj.tolist()\n    elif isinstance(obj, dict):\n        return {key: convert_numpy_to_python(value) for key, value in obj.items()}\n    elif isinstance(obj, list):\n        return [convert_numpy_to_python(element) for element in obj]\n    elif isinstance(obj, tuple):\n        return tuple(convert_numpy_to_python(element) for element in obj)\n    else:\n        return obj\n\nclass FastXGBoostPCAConfiguration:\n    \"\"\"Configuration optimized for speed\"\"\"\n    \n    def __init__(self):\n        # Model registration\n        self.model_name = \"xgboost_wide_pca_fast\"\n        self.model_directory = os.path.join(global_config.base_dir, \"xgboost_wide_pca_fast\")\n        \n        # Register model with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # Data paths from global configuration\n        self.train_path = global_config.train_path\n        self.test_path = global_config.test_path\n        self.sample_sub_path = global_config.sample_sub_path\n        \n        # SPEED OPTIMIZATIONS\n        self.chunk_size = 50000  # Larger chunks for faster I/O\n        self.sample_size_for_fitting = 20000  # Smaller sample for fitting\n        self.feature_selection_sample = 10000  # Much smaller sample\n        self.use_disk_storage = True\n        \n        # Use all CPU cores\n        self.n_jobs = mp.cpu_count()\n        \n        # Temporary storage\n        self.temp_dir = os.path.join(self.model_directory, 'temp')\n        Path(self.temp_dir).mkdir(parents=True, exist_ok=True)\n        \n        # Feature selection parameters (more aggressive for speed)\n        self.variance_threshold = 0.02  # Higher threshold\n        self.n_top_features = 200  # Fewer features for speed\n        \n        # PCA parameters - using TruncatedSVD for speed\n        self.pca_configs = [\n            {\"n_components\": 50, \"name\": \"svd_50\"},\n            {\"n_components\": 75, \"name\": \"svd_75\"}  # Reduced configs\n        ]\n        \n        # XGBoost parameters (optimized for speed)\n        self.xgb_params = {\n            \"tree_method\": \"hist\",  # Fast histogram method\n            \"device\": \"cpu\",\n            \"colsample_bytree\": 0.8,\n            \"gamma\": 0.5,\n            \"learning_rate\": 0.1,  # Higher learning rate\n            \"max_depth\": 5,  # Shallower trees\n            \"min_child_weight\": 10,\n            \"n_estimators\": 200,  # Fewer trees\n            \"n_jobs\": self.n_jobs,\n            \"random_state\": 42,\n            \"reg_alpha\": 1.0,\n            \"reg_lambda\": 5.0,\n            \"subsample\": 0.8,\n            \"verbosity\": 0,\n            \"eval_metric\": \"rmse\"\n        }\n        \n        # Validation instead of CV for speed\n        self.validation_size = 0.2\n        self.random_state = 42\n        self.early_stopping_rounds = 20\n        \n        # Output paths\n        self.submission_file = os.path.join(self.model_directory, \"submission.csv\")\n        self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n        self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n\nclass FastDataProcessor:\n    \"\"\"Fast data processing with larger chunks and parallel processing\"\"\"\n    \n    def __init__(self, config: FastXGBoostPCAConfiguration):\n        self.config = config\n    \n    def read_parquet_sample(self, filepath: str, n_rows: int, columns: List[str] = None) -> pd.DataFrame:\n        \"\"\"Read a sample of parquet file efficiently\"\"\"\n        parquet_file = pq.ParquetFile(filepath)\n        \n        # Calculate how many row groups to read\n        total_rows = parquet_file.metadata.num_rows\n        if n_rows >= total_rows:\n            return parquet_file.read(columns=columns).to_pandas()\n        \n        # Read proportionally from row groups\n        n_row_groups = parquet_file.num_row_groups\n        rows_per_group = total_rows // n_row_groups\n        groups_to_read = max(1, (n_rows // rows_per_group))\n        \n        # Read selected row groups\n        row_groups = list(range(0, n_row_groups, max(1, n_row_groups // groups_to_read)))[:groups_to_read]\n        table = parquet_file.read_row_groups(row_groups, columns=columns)\n        df = table.to_pandas()\n        \n        # Sample if we got too many rows\n        if len(df) > n_rows:\n            df = df.sample(n=n_rows, random_state=42)\n        \n        return df\n    \n    def get_feature_columns(self, filepath: str) -> List[str]:\n        \"\"\"Get feature columns quickly\"\"\"\n        parquet_file = pq.ParquetFile(filepath)\n        columns = parquet_file.schema.names\n        exclude_cols = ['label', 'timestamp', 'original_index', '__index_level_0__']\n        \n        # Read tiny sample to identify numeric columns\n        sample = self.read_parquet_sample(filepath, 100)\n        numeric_cols = sample.select_dtypes(include=[np.number]).columns.tolist()\n        \n        return [col for col in numeric_cols if col not in exclude_cols]\n    \n    def clean_data_fast(self, df: pd.DataFrame) -> pd.DataFrame:\n        \"\"\"Fast data cleaning\"\"\"\n        # Replace infinities\n        df = df.replace([np.inf, -np.inf], np.nan)\n        \n        # Fast median imputation\n        medians = df.median()\n        df = df.fillna(medians)\n        \n        # Simple clipping using quantiles\n        for col in df.columns:\n            q1, q99 = df[col].quantile([0.01, 0.99])\n            df[col] = df[col].clip(lower=q1, upper=q99)\n        \n        return df\n    \n    def select_features_fast(self, train_path: str) -> List[str]:\n        \"\"\"Fast feature selection\"\"\"\n        print(f\"Fast feature selection on {self.config.feature_selection_sample} rows...\")\n        \n        # Get all feature columns\n        feature_cols = self.get_feature_columns(train_path)\n        print(f\"Initial features: {len(feature_cols)}\")\n        \n        # Read sample\n        sample_df = self.read_parquet_sample(train_path, self.config.feature_selection_sample, feature_cols)\n        sample_clean = self.clean_data_fast(sample_df)\n        \n        # Fast variance threshold\n        selector = VarianceThreshold(threshold=self.config.variance_threshold)\n        selector.fit(sample_clean)\n        high_var_features = sample_clean.columns[selector.get_support()].tolist()\n        print(f\"After variance filter: {len(high_var_features)}\")\n        \n        # Take top features by variance\n        if len(high_var_features) > self.config.n_top_features:\n            variances = sample_clean[high_var_features].var()\n            selected_features = variances.nlargest(self.config.n_top_features).index.tolist()\n        else:\n            selected_features = high_var_features\n        \n        print(f\"Final selected features: {len(selected_features)}\")\n        \n        # Clean up\n        del sample_df, sample_clean\n        gc.collect()\n        \n        return selected_features\n\nclass FastPCATransformer:\n    \"\"\"Fast PCA using TruncatedSVD\"\"\"\n    \n    def __init__(self, config: FastXGBoostPCAConfiguration):\n        self.config = config\n        self.processor = FastDataProcessor(config)\n        self.scalers = {}\n        self.svd_models = {}\n    \n    def fit_svd_on_sample(self, train_path: str, selected_features: List[str], \n                          pca_config: Dict[str, Any]) -> Tuple[Any, Any]:\n        \"\"\"Fit scaler and SVD quickly on sample\"\"\"\n        name = pca_config[\"name\"]\n        n_components = pca_config[\"n_components\"]\n        \n        print(f\"Fitting {name} on sample...\")\n        \n        # Read sample\n        sample_df = self.processor.read_parquet_sample(\n            train_path, self.config.sample_size_for_fitting, selected_features\n        )\n        sample_clean = self.processor.clean_data_fast(sample_df)\n        \n        # Use StandardScaler (faster than RobustScaler)\n        scaler = StandardScaler()\n        sample_scaled = scaler.fit_transform(sample_clean)\n        \n        # Use TruncatedSVD (much faster than PCA)\n        svd = TruncatedSVD(n_components=n_components, random_state=42)\n        svd.fit(sample_scaled)\n        \n        # Store models\n        self.scalers[name] = scaler\n        self.svd_models[name] = svd\n        \n        # Save to disk\n        scaler_path = os.path.join(self.config.temp_dir, f'scaler_{name}.pkl')\n        svd_path = os.path.join(self.config.temp_dir, f'svd_{name}.pkl')\n        joblib.dump(scaler, scaler_path)\n        joblib.dump(svd, svd_path)\n        \n        print(f\"  Components: {svd.n_components}, Explained variance: {np.sum(svd.explained_variance_ratio_):.3f}\")\n        \n        # Clean up\n        del sample_df, sample_clean, sample_scaled\n        gc.collect()\n        \n        return scaler, svd\n    \n    def transform_data_fast(self, data_path: str, selected_features: List[str],\n                           pca_config: Dict[str, Any], is_train: bool = True) -> Tuple[np.ndarray, np.ndarray]:\n        \"\"\"Transform data quickly and return arrays directly\"\"\"\n        name = pca_config[\"name\"]\n        \n        # Load models\n        if name not in self.scalers:\n            scaler_path = os.path.join(self.config.temp_dir, f'scaler_{name}.pkl')\n            svd_path = os.path.join(self.config.temp_dir, f'svd_{name}.pkl')\n            self.scalers[name] = joblib.load(scaler_path)\n            self.svd_models[name] = joblib.load(svd_path)\n        \n        scaler = self.scalers[name]\n        svd = self.svd_models[name]\n        \n        print(f\"Transforming {'train' if is_train else 'test'} data for {name}...\")\n        \n        # Read data in larger chunks\n        parquet_file = pq.ParquetFile(data_path)\n        total_rows = parquet_file.metadata.num_rows\n        \n        # Prepare columns to read\n        columns_to_read = selected_features.copy()\n        if is_train and 'label' in parquet_file.schema.names:\n            columns_to_read.append('label')\n        \n        # Process in large batches\n        transformed_chunks = []\n        label_chunks = [] if is_train else None\n        \n        for batch in parquet_file.iter_batches(batch_size=self.config.chunk_size, columns=columns_to_read):\n            batch_df = batch.to_pandas()\n            \n            # Extract labels if training\n            if is_train and 'label' in batch_df.columns:\n                label_chunks.append(batch_df['label'].values)\n            \n            # Clean and transform\n            feature_cols = [col for col in selected_features if col in batch_df.columns]\n            batch_clean = self.processor.clean_data_fast(batch_df[feature_cols])\n            batch_scaled = scaler.transform(batch_clean)\n            batch_transformed = svd.transform(batch_scaled)\n            \n            transformed_chunks.append(batch_transformed)\n            \n            # Clean up\n            del batch_df, batch_clean, batch_scaled\n            gc.collect()\n        \n        # Concatenate results\n        X_transformed = np.vstack(transformed_chunks)\n        y = np.concatenate(label_chunks) if is_train else None\n        \n        print(f\"  Transformed shape: {X_transformed.shape}\")\n        \n        return X_transformed, y\n\nclass FastXGBoostTrainer:\n    \"\"\"Fast XGBoost training with validation set instead of CV\"\"\"\n    \n    def __init__(self, config: FastXGBoostPCAConfiguration):\n        self.config = config\n    \n    def train_model(self, X_train: np.ndarray, y_train: np.ndarray, \n                   X_valid: np.ndarray, y_valid: np.ndarray) -> Tuple[Any, np.ndarray, float]:\n        \"\"\"Train a single model quickly\"\"\"\n        # Use DMatrix for faster training\n        dtrain = xgb.DMatrix(X_train, label=y_train)\n        dvalid = xgb.DMatrix(X_valid, label=y_valid)\n        \n        # Train with early stopping\n        model = xgb.train(\n            self.config.xgb_params,\n            dtrain,\n            num_boost_round=self.config.xgb_params['n_estimators'],\n            evals=[(dvalid, 'valid')],\n            early_stopping_rounds=self.config.early_stopping_rounds,\n            verbose_eval=False\n        )\n        \n        # Get predictions\n        valid_pred = model.predict(dvalid)\n        score = pearsonr(y_valid, valid_pred)[0]\n        \n        return model, valid_pred, score\n\nclass FastXGBoostPCAPipeline:\n    \"\"\"Main pipeline optimized for speed\"\"\"\n    \n    def __init__(self):\n        self.config = FastXGBoostPCAConfiguration()\n        self.processor = FastDataProcessor(self.config)\n        self.pca_transformer = FastPCATransformer(self.config)\n        self.trainer = FastXGBoostTrainer(self.config)\n    \n    def run_pipeline(self) -> float:\n        \"\"\"Execute the fast pipeline\"\"\"\n        print(\"\\nStarting Fast XGBoost Wide Features with PCA Pipeline\")\n        print(\"=\"*80)\n        \n        # Update model status\n        global_config.update_model_status(self.config.model_name, \"running\")\n        \n        try:\n            # Step 1: Fast feature selection\n            selected_features = self.processor.select_features_fast(self.config.train_path)\n            \n            # Save selected features\n            feature_list_path = os.path.join(self.config.model_directory, 'selected_features.json')\n            with open(feature_list_path, 'w') as f:\n                json.dump(selected_features, f, indent=2)\n            \n            # Step 2: Fit SVD models\n            for pca_config in self.config.pca_configs:\n                self.pca_transformer.fit_svd_on_sample(\n                    self.config.train_path, selected_features, pca_config\n                )\n            \n            # Step 3: Transform data (in memory, much faster)\n            print(\"\\nTransforming data...\")\n            transformed_data = {}\n            \n            for pca_config in self.config.pca_configs:\n                # Transform training data\n                X_train_full, y_train_full = self.pca_transformer.transform_data_fast(\n                    self.config.train_path, selected_features, pca_config, is_train=True\n                )\n                \n                # Transform test data\n                X_test, _ = self.pca_transformer.transform_data_fast(\n                    self.config.test_path, selected_features, pca_config, is_train=False\n                )\n                \n                transformed_data[pca_config['name']] = {\n                    'X_train': X_train_full,\n                    'y_train': y_train_full,\n                    'X_test': X_test\n                }\n            \n            # Step 4: Train with validation split (much faster than CV)\n            print(\"\\nTraining models...\")\n            all_scores = {}\n            all_test_predictions = {}\n            all_valid_predictions = {}\n            \n            # Create train/validation split once\n            n_samples = len(y_train_full)\n            indices = np.arange(n_samples)\n            train_idx, valid_idx = train_test_split(\n                indices, \n                test_size=self.config.validation_size,\n                random_state=self.config.random_state,\n                shuffle=True\n            )\n            \n            for pca_name, data in transformed_data.items():\n                print(f\"\\nTraining model for {pca_name}...\")\n                \n                # Split data\n                X_train = data['X_train'][train_idx]\n                y_train = data['y_train'][train_idx]\n                X_valid = data['X_train'][valid_idx]\n                y_valid = data['y_train'][valid_idx]\n                \n                # Train model\n                model, valid_pred, score = self.trainer.train_model(\n                    X_train, y_train, X_valid, y_valid\n                )\n                \n                print(f\"  Validation score: {score:.4f}\")\n                \n                # Predict on test\n                dtest = xgb.DMatrix(data['X_test'])\n                test_pred = model.predict(dtest)\n                \n                # Store results\n                all_scores[pca_name] = {\n                    'validation_score': score,\n                    'n_trees': model.best_iteration\n                }\n                all_test_predictions[pca_name] = test_pred\n                all_valid_predictions[pca_name] = (valid_idx, valid_pred)\n                \n                # Clean up\n                del model, X_train, y_train, X_valid, y_valid\n                gc.collect()\n            \n            # Step 5: Create ensemble\n            print(\"\\nCreating ensemble...\")\n            \n            # Calculate weights based on validation scores\n            scores = np.array([all_scores[name]['validation_score'] for name in all_scores])\n            weights = scores / scores.sum()\n            \n            # Create OOF predictions array\n            oof_predictions = np.zeros(n_samples)\n            for pca_name, (v_idx, v_pred) in all_valid_predictions.items():\n                weight = weights[list(all_scores.keys()).index(pca_name)]\n                oof_predictions[v_idx] += weight * v_pred\n            \n            # Weighted average for test\n            test_ensemble = np.average(\n                list(all_test_predictions.values()),\n                axis=0,\n                weights=weights\n            )\n            \n            # Calculate ensemble score on validation data\n            ensemble_score = pearsonr(y_train_full[valid_idx], oof_predictions[valid_idx])[0]\n            print(f\"Ensemble validation score: {ensemble_score:.4f}\")\n            \n            # Step 6: Save outputs\n            print(\"\\nSaving outputs...\")\n            \n            # Save submission\n            sample_submission = pd.read_csv(self.config.sample_sub_path)\n            sample_submission['prediction'] = test_ensemble\n            sample_submission.to_csv(self.config.submission_file, index=False)\n            \n            # Save OOF predictions (validation only)\n            oof_df = pd.DataFrame({\n                'row_id': valid_idx,\n                'true_label': y_train_full[valid_idx],\n                'prediction': oof_predictions[valid_idx]\n            })\n            oof_df.to_csv(self.config.oof_predictions_path, index=False)\n            \n            # Save model performance - convert numpy types to Python native types\n            performance_data = {\n                'ensemble_score': float(ensemble_score),\n                'model_scores': convert_numpy_to_python(all_scores),\n                'weights': convert_numpy_to_python({\n                    name: float(w) for name, w in zip(all_scores.keys(), weights)\n                }),\n                'n_features': int(len(selected_features)),\n                'n_train_samples': int(len(train_idx)),\n                'n_valid_samples': int(len(valid_idx))\n            }\n            \n            with open(self.config.model_performance_path, 'w') as f:\n                json.dump(performance_data, f, indent=2)\n            \n            # Register outputs with global config\n            global_config.register_model_output(\n                self.config.model_name,\n                'submission',\n                self.config.submission_file,\n                'submission',\n                metadata={'ensemble_score': float(ensemble_score)}\n            )\n            \n            global_config.register_model_output(\n                self.config.model_name,\n                'oof_predictions',\n                self.config.oof_predictions_path,\n                'oof_predictions'\n            )\n            \n            # Update status\n            global_config.update_model_status(self.config.model_name, 'completed', score=ensemble_score)\n            \n            print(\"\\n✅ Fast XGBoost PCA pipeline completed successfully\")\n            print(f\"Final ensemble score: {ensemble_score:.4f}\")\n            \n            return ensemble_score\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n            print(f\"\\n❌ Pipeline failed: {error_msg}\")\n            raise\n            \n        finally:\n            # Clean up temporary files\n            if os.path.exists(self.config.temp_dir):\n                import shutil\n                shutil.rmtree(self.config.temp_dir)\n            \n            # Force garbage collection\n            gc.collect()\n\n# Main execution\nif __name__ == \"__main__\":\n    try:\n        print(\"\\n🚀 Running Fast XGBoost Wide Features with PCA Pipeline\")\n        print(\"Expected runtime: 30-60 minutes (vs 10 hours)\")\n        print(\"-\"*80)\n        \n        # Force garbage collection before starting\n        gc.collect()\n        \n        # Create and run pipeline\n        pipeline = FastXGBoostPCAPipeline()\n        final_score = pipeline.run_pipeline()\n        \n    except Exception as e:\n        print(f\"Pipeline error: {e}\")\n        raise","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# XGBOOST/LIGHTGBM TIME-WEIGHTED ENSEMBLE PIPELINE\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - XGBoost/LightGBM Time-Weighted Ensemble\nUses time decay weights and multiple time slices for improved predictions\n\"\"\"\n\nimport subprocess\nimport sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom scipy.stats import pearsonr\nfrom typing import Dict, List, Tuple, Any\nfrom pathlib import Path\nimport json\nimport gc\nimport warnings\nimport os\n\nwarnings.filterwarnings('ignore')\n\n# Install required packages\nprint(\"Installing packages for XGBoost/LightGBM Ensemble Pipeline...\")\npackages_to_install = [\n    'xgboost==2.0.3',\n    'lightgbm>=3.3.0',\n    'scikit-learn>=1.0.0'\n]\n\nfor package in packages_to_install:\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\n\n\nclass TimeWeightedEnsembleConfiguration:\n    \"\"\"Configuration for time-weighted ensemble pipeline\"\"\"\n    \n    def __init__(self):\n        # Model registration\n        self.model_name = \"xgb_lgbm_time_ensemble\"\n        self.model_directory = os.path.join(global_config.base_dir, \"xgb_lgbm_time_ensemble\")\n        \n        # Register model with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # Data paths from global configuration\n        self.train_path = global_config.train_path\n        self.test_path = global_config.test_path\n        self.sample_sub_path = global_config.sample_sub_path\n        \n        # Feature configuration\n        self.features = [\n            \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\",\n            \"X860\", \"X674\", \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\",\n            \"X178\", \"X532\", \"X168\", \"X612\",\n            \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n        ]\n        self.label_column = \"label\"\n        \n        # Cross-validation parameters\n        self.n_folds = 3\n        self.random_state = 42\n        \n        # Time decay parameters\n        self.decay_rate = 0.95\n        \n        # Model slices configuration\n        self.model_slices = [\n            {\"name\": \"full_data\", \"cutoff_pct\": 0.0},\n            {\"name\": \"last_75pct\", \"cutoff_pct\": 0.25},\n            {\"name\": \"last_50pct\", \"cutoff_pct\": 0.50}\n        ]\n        \n        # XGBoost parameters\n        self.xgb_params = {\n            \"tree_method\": \"hist\",\n            \"device\": \"cpu\",  # Changed from gpu for compatibility\n            \"colsample_bylevel\": 0.4778,\n            \"colsample_bynode\": 0.3628,\n            \"colsample_bytree\": 0.7107,\n            \"gamma\": 1.7095,\n            \"learning_rate\": 0.02213,\n            \"max_depth\": 20,\n            \"max_leaves\": 12,\n            \"min_child_weight\": 16,\n            \"n_estimators\": 1667,\n            \"subsample\": 0.06567,\n            \"reg_alpha\": 39.3524,\n            \"reg_lambda\": 75.4484,\n            \"verbosity\": 0,\n            \"random_state\": self.random_state,\n            \"n_jobs\": -1,\n            \"verbose\": False,\n        }\n        \n        # LightGBM parameters\n        self.lgbm_params = {\n            \"boosting_type\": \"gbdt\",\n            \"device\": \"cpu\",  # Changed from gpu for compatibility\n            \"n_jobs\": -1,\n            \"verbosity\": -1,  # Silent mode\n            \"random_state\": self.random_state,\n            \"colsample_bytree\": 0.5039,\n            \"learning_rate\": 0.01260,\n            \"min_child_samples\": 20,\n            \"min_child_weight\": 0.1146,\n            \"n_estimators\": 915,\n            \"num_leaves\": 145,\n            \"reg_alpha\": 19.2447,\n            \"reg_lambda\": 55.5046,\n            \"subsample\": 0.9709,\n            \"max_depth\": 9\n        }\n        \n        # Learner configuration\n        self.learners = [\n            {\"name\": \"xgb\", \"estimator\": XGBRegressor, \"params\": self.xgb_params},\n            {\"name\": \"lgbm\", \"estimator\": LGBMRegressor, \"params\": self.lgbm_params}\n        ]\n        \n        # Output paths\n        self.submission_file = os.path.join(self.model_directory, \"submission.csv\")\n        self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n        self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n        self.slice_predictions_path = os.path.join(self.model_directory, \"slice_predictions.csv\")\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n\n\nclass TimeWeightedDataProcessor:\n    \"\"\"Process data with time decay weights\"\"\"\n    \n    def __init__(self, config: TimeWeightedEnsembleConfiguration):\n        self.config = config\n    \n    def create_time_decay_weights(self, n: int, decay: float = None) -> np.ndarray:\n        \"\"\"Create time decay weights for samples\"\"\"\n        if decay is None:\n            decay = self.config.decay_rate\n            \n        positions = np.arange(n)\n        normalized = positions / float(n - 1)\n        weights = decay ** (1.0 - normalized)\n        return weights * n / weights.sum()\n    \n    def load_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n        \"\"\"Load train, test, and submission data\"\"\"\n        print(\"Loading data...\")\n        \n        train_df = pd.read_parquet(\n            self.config.train_path,\n            columns=self.config.features + [self.config.label_column]\n        ).reset_index(drop=True)\n        \n        test_df = pd.read_parquet(\n            self.config.test_path,\n            columns=self.config.features\n        ).reset_index(drop=True)\n        \n        submission_df = pd.read_csv(self.config.sample_sub_path)\n        \n        print(f\"Loaded train: {train_df.shape}, test: {test_df.shape}, submission: {submission_df.shape}\")\n        \n        return train_df, test_df, submission_df\n    \n    def prepare_model_slices(self, n_samples: int) -> List[Dict[str, Any]]:\n        \"\"\"Prepare model slices with actual cutoff indices\"\"\"\n        slices = []\n        for slice_config in self.config.model_slices:\n            cutoff = int(slice_config[\"cutoff_pct\"] * n_samples)\n            slices.append({\n                \"name\": slice_config[\"name\"],\n                \"cutoff\": cutoff,\n                \"cutoff_pct\": slice_config[\"cutoff_pct\"]\n            })\n        return slices\n\n\nclass TimeWeightedModelTrainer:\n    \"\"\"Train models with time weighting and multiple slices\"\"\"\n    \n    def __init__(self, config: TimeWeightedEnsembleConfiguration):\n        self.config = config\n        self.processor = TimeWeightedDataProcessor(config)\n    \n    def train_fold(self, train_df: pd.DataFrame, train_idx: np.ndarray, \n                   valid_idx: np.ndarray, model_slices: List[Dict],\n                   full_weights: np.ndarray, fold: int) -> Dict[str, Dict[str, np.ndarray]]:\n        \"\"\"Train all models for a single fold\"\"\"\n        \n        X_valid = train_df.iloc[valid_idx][self.config.features]\n        y_valid = train_df.iloc[valid_idx][self.config.label_column]\n        \n        fold_predictions = {\n            learner[\"name\"]: {slice_info[\"name\"]: np.zeros(len(valid_idx)) \n                            for slice_info in model_slices}\n            for learner in self.config.learners\n        }\n        \n        for slice_info in model_slices:\n            slice_name = slice_info[\"name\"]\n            cutoff = slice_info[\"cutoff\"]\n            \n            # Prepare training data for this slice\n            subset = train_df.iloc[cutoff:].reset_index(drop=True)\n            rel_idx = train_idx[train_idx >= cutoff] - cutoff\n            \n            X_train = subset.iloc[rel_idx][self.config.features]\n            y_train = subset.iloc[rel_idx][self.config.label_column]\n            \n            # Calculate sample weights\n            if cutoff == 0:\n                sw = full_weights[train_idx]\n            else:\n                sw_total = self.processor.create_time_decay_weights(len(subset))\n                sw = sw_total[rel_idx]\n            \n            # Train each learner\n            for learner in self.config.learners:\n                name = learner[\"name\"]\n                EstimatorClass = learner[\"estimator\"]\n                params = learner[\"params\"].copy()\n                \n                print(f\"  Training {name} on {slice_name} (fold {fold})...\")\n                \n                # Create and train model\n                model = EstimatorClass(**params)\n                \n                # Handle different eval_set signatures\n                if name == \"xgb\":\n                    model.fit(X_train, y_train, sample_weight=sw,\n                             eval_set=[(X_valid, y_valid)], verbose=False)\n                else:  # lgbm\n                    model.fit(X_train, y_train, sample_weight=sw,\n                             eval_set=[(X_valid, y_valid)], \n                             eval_metric='rmse',\n                             callbacks=[lgb.log_evaluation(0)])\n                \n                # Make predictions\n                mask = valid_idx >= cutoff\n                if mask.any():\n                    idxs = valid_idx[mask]\n                    local_idxs = np.where(mask)[0]\n                    preds = model.predict(train_df.iloc[idxs][self.config.features])\n                    fold_predictions[name][slice_name][local_idxs] = preds\n                \n                # For samples before cutoff, use full_data predictions\n                if cutoff > 0 and (~mask).any():\n                    local_idxs = np.where(~mask)[0]\n                    fold_predictions[name][slice_name][local_idxs] = (\n                        fold_predictions[name][\"full_data\"][local_idxs]\n                    )\n                \n                # Clean up\n                del model\n                gc.collect()\n        \n        return fold_predictions\n    \n    def predict_test(self, train_df: pd.DataFrame, test_df: pd.DataFrame,\n                     model_slices: List[Dict], full_weights: np.ndarray) -> Dict[str, Dict[str, np.ndarray]]:\n        \"\"\"Make predictions on test set using all folds\"\"\"\n        \n        n_test = len(test_df)\n        test_predictions = {\n            learner[\"name\"]: {slice_info[\"name\"]: np.zeros(n_test) \n                            for slice_info in model_slices}\n            for learner in self.config.learners\n        }\n        \n        # KFold for consistency with training\n        kf = KFold(n_splits=self.config.n_folds, shuffle=False)\n        \n        for fold, (train_idx, _) in enumerate(kf.split(train_df), start=1):\n            print(f\"\\nGenerating test predictions for fold {fold}...\")\n            \n            for slice_info in model_slices:\n                slice_name = slice_info[\"name\"]\n                cutoff = slice_info[\"cutoff\"]\n                \n                # Prepare training data\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                \n                X_train = subset.iloc[rel_idx][self.config.features]\n                y_train = subset.iloc[rel_idx][self.config.label_column]\n                \n                # Calculate sample weights\n                if cutoff == 0:\n                    sw = full_weights[train_idx]\n                else:\n                    sw_total = self.processor.create_time_decay_weights(len(subset))\n                    sw = sw_total[rel_idx]\n                \n                # Train and predict for each learner\n                for learner in self.config.learners:\n                    name = learner[\"name\"]\n                    EstimatorClass = learner[\"estimator\"]\n                    params = learner[\"params\"].copy()\n                    \n                    # Train model\n                    model = EstimatorClass(**params)\n                    if name == \"xgb\":\n                        model.fit(X_train, y_train, sample_weight=sw, verbose=False)\n                    else:  # lgbm\n                        # For test predictions, no eval set needed\n                        model.fit(X_train, y_train, sample_weight=sw,\n                                callbacks=[lgb.log_evaluation(0)])\n                    \n                    # Predict on test\n                    test_predictions[name][slice_name] += model.predict(test_df[self.config.features])\n                    \n                    # Clean up\n                    del model\n                    gc.collect()\n        \n        # Average across folds\n        for name in test_predictions:\n            for slice_name in test_predictions[name]:\n                test_predictions[name][slice_name] /= self.config.n_folds\n        \n        return test_predictions\n\n\nclass TimeWeightedEnsemblePipeline:\n    \"\"\"Main pipeline for time-weighted ensemble\"\"\"\n    \n    def __init__(self):\n        self.config = TimeWeightedEnsembleConfiguration()\n        self.processor = TimeWeightedDataProcessor(self.config)\n        self.trainer = TimeWeightedModelTrainer(self.config)\n    \n    def compute_ensemble_predictions(self, oof_preds: Dict, test_preds: Dict,\n                                   train_labels: np.ndarray) -> Tuple[Dict, Dict, Dict]:\n        \"\"\"Compute various ensemble predictions and scores\"\"\"\n        \n        # Compute Pearson scores per learner and slice\n        pearson_scores = {\n            name: {slice_name: pearsonr(train_labels, preds)[0]\n                  for slice_name, preds in slices.items()}\n            for name, slices in oof_preds.items()\n        }\n        \n        print(\"\\nPearson scores by learner and slice:\")\n        for learner, scores in pearson_scores.items():\n            print(f\"{learner}:\")\n            for slice_name, score in scores.items():\n                print(f\"  {slice_name}: {score:.4f}\")\n        \n        # Ensemble per learner across slices\n        learner_ensembles = {}\n        ensemble_scores = {}\n        \n        for learner_name, slice_scores in pearson_scores.items():\n            # Simple average ensemble\n            oof_simple = np.mean(list(oof_preds[learner_name].values()), axis=0)\n            test_simple = np.mean(list(test_preds[learner_name].values()), axis=0)\n            score_simple = pearsonr(train_labels, oof_simple)[0]\n            \n            # Weighted ensemble (by score)\n            total_score = sum(slice_scores.values())\n            slice_weights = {sn: sc/total_score for sn, sc in slice_scores.items()}\n            oof_weighted = sum(slice_weights[sn] * oof_preds[learner_name][sn]\n                             for sn in slice_weights)\n            test_weighted = sum(slice_weights[sn] * test_preds[learner_name][sn]\n                              for sn in slice_weights)\n            score_weighted = pearsonr(train_labels, oof_weighted)[0]\n            \n            print(f\"\\n{learner_name.upper()} Simple ensemble Pearson:   {score_simple:.4f}\")\n            print(f\"{learner_name.upper()} Weighted ensemble Pearson: {score_weighted:.4f}\")\n            \n            learner_ensembles[learner_name] = {\n                \"oof_simple\": oof_simple,\n                \"test_simple\": test_simple,\n                \"oof_weighted\": oof_weighted,\n                \"test_weighted\": test_weighted\n            }\n            \n            ensemble_scores[learner_name] = {\n                \"simple\": score_simple,\n                \"weighted\": score_weighted\n            }\n        \n        return pearson_scores, learner_ensembles, ensemble_scores\n    \n    def run_pipeline(self) -> float:\n        \"\"\"Execute the time-weighted ensemble pipeline\"\"\"\n        print(\"\\nStarting XGBoost/LightGBM Time-Weighted Ensemble Pipeline\")\n        print(\"=\"*80)\n        \n        # Update model status\n        global_config.update_model_status(self.config.model_name, \"running\")\n        \n        try:\n            # Load data\n            train_df, test_df, submission_df = self.processor.load_data()\n            n_samples = len(train_df)\n            \n            # Prepare model slices\n            model_slices = self.processor.prepare_model_slices(n_samples)\n            \n            # Create time decay weights\n            full_weights = self.processor.create_time_decay_weights(n_samples)\n            \n            # Initialize storage for predictions\n            oof_preds = {\n                learner[\"name\"]: {sl[\"name\"]: np.zeros(n_samples) for sl in model_slices}\n                for learner in self.config.learners\n            }\n            \n            # Cross-validation\n            kf = KFold(n_splits=self.config.n_folds, shuffle=False)\n            \n            print(\"\\nRunning cross-validation...\")\n            for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n                print(f\"\\n--- Fold {fold}/{self.config.n_folds} ---\")\n                \n                # Train models for this fold\n                fold_predictions = self.trainer.train_fold(\n                    train_df, train_idx, valid_idx, model_slices, full_weights, fold\n                )\n                \n                # Store OOF predictions\n                for learner_name, slice_preds in fold_predictions.items():\n                    for slice_name, preds in slice_preds.items():\n                        oof_preds[learner_name][slice_name][valid_idx] = preds\n            \n            # Get test predictions\n            print(\"\\nGenerating final test predictions...\")\n            test_preds = self.trainer.predict_test(train_df, test_df, model_slices, full_weights)\n            \n            # Compute ensembles\n            train_labels = train_df[self.config.label_column].values\n            pearson_scores, learner_ensembles, ensemble_scores = self.compute_ensemble_predictions(\n                oof_preds, test_preds, train_labels\n            )\n            \n            # Final ensemble across learners\n            final_oof = np.mean([le[\"oof_simple\"] for le in learner_ensembles.values()], axis=0)\n            final_test = np.mean([le[\"test_simple\"] for le in learner_ensembles.values()], axis=0)\n            final_score = pearsonr(train_labels, final_oof)[0]\n            \n            print(f\"\\nFINAL ensemble across learners Pearson: {final_score:.4f}\")\n            \n            # Save outputs\n            print(\"\\nSaving outputs...\")\n            \n            # Save submission\n            submission_df[\"prediction\"] = final_test\n            submission_df.to_csv(self.config.submission_file, index=False)\n            \n            # Save OOF predictions with all details\n            oof_df = pd.DataFrame({\n                'row_id': np.arange(n_samples),\n                'true_label': train_labels,\n                'final_prediction': final_oof\n            })\n            \n            # Add individual learner predictions\n            for learner_name in oof_preds:\n                oof_df[f'{learner_name}_simple'] = learner_ensembles[learner_name][\"oof_simple\"]\n                oof_df[f'{learner_name}_weighted'] = learner_ensembles[learner_name][\"oof_weighted\"]\n            \n            # Add slice predictions\n            for learner_name, slices in oof_preds.items():\n                for slice_name, preds in slices.items():\n                    oof_df[f'{learner_name}_{slice_name}'] = preds\n            \n            oof_df.to_csv(self.config.oof_predictions_path, index=False)\n            \n            # Save model performance\n            performance_data = {\n                'final_ensemble_score': float(final_score),\n                'learner_scores': {\n                    learner: {\n                        'slices': {k: float(v) for k, v in scores.items()},\n                        'ensemble': {k: float(v) for k, v in ensemble_scores[learner].items()}\n                    }\n                    for learner, scores in pearson_scores.items()\n                },\n                'n_features': len(self.config.features),\n                'n_samples': n_samples,\n                'model_slices': model_slices,\n                'decay_rate': self.config.decay_rate\n            }\n            \n            with open(self.config.model_performance_path, 'w') as f:\n                json.dump(performance_data, f, indent=2)\n            \n            # Register outputs with global config\n            global_config.register_model_output(\n                self.config.model_name,\n                'submission',\n                self.config.submission_file,\n                'submission',\n                metadata={'ensemble_score': float(final_score)}\n            )\n            \n            global_config.register_model_output(\n                self.config.model_name,\n                'oof_predictions',\n                self.config.oof_predictions_path,\n                'oof_predictions'\n            )\n            \n            # Update status\n            global_config.update_model_status(\n                self.config.model_name, \n                'completed', \n                score=final_score\n            )\n            \n            print(\"\\n✅ XGBoost/LightGBM Time-Weighted Ensemble pipeline completed successfully\")\n            print(f\"Final ensemble score: {final_score:.4f}\")\n            \n            return final_score\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(\n                self.config.model_name, \n                'failed', \n                error_message=error_msg\n            )\n            print(f\"\\n❌ Pipeline failed: {error_msg}\")\n            raise\n        \n        finally:\n            # Force garbage collection\n            gc.collect()\n\n\n# Main execution\nif __name__ == \"__main__\":\n    try:\n        print(\"\\n🚀 Running XGBoost/LightGBM Time-Weighted Ensemble Pipeline\")\n        print(\"-\"*80)\n        \n        # Force garbage collection before starting\n        gc.collect()\n        \n        # Create and run pipeline\n        pipeline = TimeWeightedEnsemblePipeline()\n        final_score = pipeline.run_pipeline()\n        \n    except Exception as e:\n        print(f\"Pipeline error: {e}\")\n        raise","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# AUTOENCODER SIMPLE PIPELINE - STANDALONE IMPLEMENTATION\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - AutoEncoder Simple Pipeline\nSimple averaging ensemble version with meta-learning features\n\"\"\"\n\nimport subprocess\nimport sys\nimport os\nimport gc\nimport warnings\nimport json\nimport random\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr\nfrom typing import List, Dict, Tuple, Optional, Any\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nfrom sklearn.model_selection import TimeSeriesSplit, KFold\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nfrom datetime import datetime\n\nwarnings.filterwarnings('ignore')\n\n# Check that setup cell was run\ntry:\n    global_config\n    print(\"✓ Global configuration found\")\nexcept NameError:\n    raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# Install required packages\nprint(\"Installing packages for AutoEncoder Simple pipeline...\")\npackages_to_install = [\n    'torch==2.0.1',\n    'torchvision==0.15.2',\n    'tqdm==4.65.0'\n]\n\nprint(\"Installing PyTorch (CPU version)...\")\nsubprocess.check_call([\n    sys.executable, \"-m\", \"pip\", \"install\", \n    \"torch==2.0.1\", \"torchvision==0.15.2\",\n    \"--index-url\", \"https://download.pytorch.org/whl/cpu\",\n    \"--quiet\"\n])\n\nsubprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"tqdm==4.65.0\", \"--quiet\"])\n\n# Import tqdm after installation\nfrom tqdm import tqdm\n\n# =============================================================================\n# AUTOENCODER CONFIGURATION\n# =============================================================================\n\n@dataclass\nclass AutoEncoderSimpleConfiguration:\n    \"\"\"Configuration for AutoEncoder Simple Pipeline\"\"\"\n    \n    # Model identification\n    model_name: str = \"autoencoder_simple\"\n    model_directory: str = \"\"\n    \n    # Model-specific features\n    feature_columns: List[str] = field(default_factory=list)\n    actual_features: List[str] = field(default_factory=list)\n    \n    # AutoEncoder architecture parameters\n    encoding_size: int = 128\n    hidden_size: int = 256\n    dropout: float = 0.7\n    ae_dropout: float = 0.3\n    noise_std: float = 0.05\n    num_blocks: int = 8\n    \n    # Training parameters\n    num_epochs: int = 80\n    batch_size: int = 4096\n    learning_rate: float = 0.0001\n    weight_decay: float = 5e-3\n    patience: int = 10\n    min_lr: float = 1e-6\n    \n    # Loss weights\n    mse_weight: float = 0.25\n    corr_weight: float = 0.6\n    ae_weight: float = 0.15\n    \n    # Cross-validation parameters\n    n_splits: int = 5\n    max_train_size: int = 100_000_000\n    gap: int = 1\n    \n    # Random seed\n    seed: int = 42\n    \n    def __post_init__(self):\n        \"\"\"Initialize configuration and register model\"\"\"\n        if not self.model_directory:\n            self.model_directory = os.path.join(global_config.base_dir, \"autoencoder_simple\")\n        \n        # Register model with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # Set default features if not provided\n        if not self.feature_columns:\n            self.feature_columns = [\n                \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n                \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n                \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\",\n                \"X188\", \"X207\", \"X219\", \"X233\", \"X245\"\n            ]\n        \n        # Output paths\n        self.intermediate_dir = os.path.join(self.model_directory, \"fold_models\")\n        self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n        self.config_path = os.path.join(self.model_directory, \"config.json\")\n        self.performance_metrics_path = os.path.join(self.model_directory, \"performance_metrics.json\")\n        \n        # Meta-learning data paths\n        self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n        self.encoded_features_path = os.path.join(self.model_directory, \"encoded_features.csv\")\n        self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n        self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n        self.training_history_path = os.path.join(self.model_directory, \"training_history.json\")\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n        Path(self.intermediate_dir).mkdir(parents=True, exist_ok=True)\n    \n    def save(self):\n        \"\"\"Save configuration to JSON file\"\"\"\n        config_dict = {k: v for k, v in self.__dict__.items() \n                      if not k.startswith('_') and k not in ['model_directory', 'intermediate_dir']}\n        with open(self.config_path, 'w') as f:\n            json.dump(config_dict, f, indent=2)\n\n# =============================================================================\n# HELPER CLASSES\n# =============================================================================\n\nclass TorchEnvironmentManager:\n    @staticmethod\n    def configure_environment():\n        os.environ['CUDA_VISIBLE_DEVICES'] = ''\n        os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:128'\n        os.environ['PYTORCH_NO_CUDA_MEMORY_CACHING'] = '1'\n        os.environ['TRITON_CACHE_DIR'] = f'/tmp/triton_cache_{os.getpid()}'\n        os.environ['DISABLE_TRITON'] = '1'\n\nclass RandomSeedManager:\n    @staticmethod\n    def set_all_seeds(seed: int):\n        random.seed(seed)\n        np.random.seed(seed)\n        os.environ['PYTHONHASHSEED'] = str(seed)\n        try:\n            import torch\n            torch.manual_seed(seed)\n            torch.cuda.manual_seed(seed)\n            torch.cuda.manual_seed_all(seed)\n            torch.backends.cudnn.deterministic = True\n            torch.backends.cudnn.benchmark = False\n        except:\n            pass\n\n# =============================================================================\n# NEURAL NETWORK COMPONENTS\n# =============================================================================\n\nclass AutoEncoderNeuralNetwork:\n    \"\"\"Neural network components for AutoEncoder architecture\"\"\"\n    \n    def __init__(self, config: AutoEncoderSimpleConfiguration):\n        self.config = config\n        self.device = None\n        self.torch = None\n        self.nn = None\n        self.F = None\n    \n    def initialize_pytorch(self):\n        \"\"\"Initialize PyTorch modules after environment configuration\"\"\"\n        try:\n            # Simple import without complex cleanup\n            import torch\n            import torch.nn as nn\n            import torch.nn.functional as F\n            \n            self.torch = torch\n            self.nn = nn\n            self.F = F\n            self.device = torch.device('cpu')\n            \n            print(f\"PyTorch version: {torch.__version__}\")\n            print(f\"Using device: {self.device}\")\n            \n            return True\n            \n        except Exception as e:\n            print(f\"Error initializing PyTorch: {e}\")\n            return False\n    \n    def create_autoencoder(self, input_size: int):\n        \"\"\"Create autoencoder neural network\"\"\"\n        \n        class Swish(self.nn.Module):\n            def forward(self, x):\n                return x * self.torch.sigmoid(x)\n        \n        class GaussianNoise(self.nn.Module):\n            def __init__(self, std: float = 0.05):\n                super().__init__()\n                self.std = std\n                \n            def forward(self, x):\n                if self.training:\n                    noise = self.torch.randn_like(x) * self.std\n                    return x + noise\n                return x\n        \n        class AutoEncoder(self.nn.Module):\n            def __init__(self, input_size: int, encoding_size: int, dropout: float):\n                super().__init__()\n                \n                # Encoder layers\n                self.encoder = self.nn.Sequential(\n                    self.nn.Linear(input_size, input_size // 2),\n                    self.nn.BatchNorm1d(input_size // 2),\n                    Swish(),\n                    self.nn.Dropout(dropout),\n                    \n                    self.nn.Linear(input_size // 2, input_size // 4),\n                    self.nn.BatchNorm1d(input_size // 4),\n                    Swish(),\n                    self.nn.Dropout(dropout),\n                    \n                    self.nn.Linear(input_size // 4, encoding_size),\n                    self.nn.BatchNorm1d(encoding_size),\n                    Swish()\n                )\n                \n                # Decoder layers\n                self.decoder = self.nn.Sequential(\n                    self.nn.Linear(encoding_size, input_size // 4),\n                    self.nn.BatchNorm1d(input_size // 4),\n                    Swish(),\n                    self.nn.Dropout(dropout),\n                    \n                    self.nn.Linear(input_size // 4, input_size // 2),\n                    self.nn.BatchNorm1d(input_size // 2),\n                    Swish(),\n                    self.nn.Dropout(dropout),\n                    \n                    self.nn.Linear(input_size // 2, input_size)\n                )\n                \n            def forward(self, x):\n                encoded = self.encoder(x)\n                decoded = self.decoder(encoded)\n                return encoded, decoded\n        \n        return AutoEncoder(input_size, self.config.encoding_size, self.config.ae_dropout)\n    \n    def create_full_model(self, input_size: int):\n        \"\"\"Create complete model with autoencoder and prediction head\"\"\"\n        \n        autoencoder = self.create_autoencoder(input_size)\n        config = self.config\n        torch = self.torch\n        \n        class CryptoMLPWithAutoEncoder(self.nn.Module):\n            def __init__(self):\n                super().__init__()\n                \n                self.noise_layer = self.nn.Sequential()  # Will add GaussianNoise\n                self.autoencoder = autoencoder\n                \n                combined_input_size = input_size + config.encoding_size\n                self.input_bn = self.nn.BatchNorm1d(combined_input_size)\n                \n                # Initial block\n                self.initial_block = self.nn.Sequential(\n                    self.nn.Linear(combined_input_size, config.hidden_size),\n                    self.nn.BatchNorm1d(config.hidden_size),\n                    self.nn.ReLU(),\n                    self.nn.Dropout(config.dropout),\n                    \n                    self.nn.Linear(config.hidden_size, config.hidden_size),\n                    self.nn.BatchNorm1d(config.hidden_size),\n                    self.nn.ReLU(),\n                    self.nn.Dropout(config.dropout),\n                    \n                    self.nn.Linear(config.hidden_size, config.hidden_size),\n                    self.nn.BatchNorm1d(config.hidden_size),\n                    self.nn.ReLU()\n                )\n                \n                # Residual blocks\n                self.residual_blocks = self.nn.ModuleList()\n                for _ in range(config.num_blocks):\n                    block = self.nn.Sequential(\n                        self.nn.Linear(config.hidden_size, config.hidden_size),\n                        self.nn.BatchNorm1d(config.hidden_size),\n                        self.nn.ReLU(),\n                        self.nn.Dropout(config.dropout),\n                        \n                        self.nn.Linear(config.hidden_size, config.hidden_size),\n                        self.nn.BatchNorm1d(config.hidden_size),\n                        self.nn.ReLU()\n                    )\n                    self.residual_blocks.append(block)\n                \n                # Output layer\n                self.output = self.nn.Linear(config.hidden_size, 1)\n                \n            def forward(self, x, return_ae_loss=False, return_encodings=False):\n                # Add noise during training\n                if self.training:\n                    x = x + torch.randn_like(x) * config.noise_std\n                \n                # Autoencoder\n                encoded, decoded = self.autoencoder(x)\n                \n                # Combine original and encoded features\n                x_combined = torch.cat([x, encoded], dim=1)\n                x_combined = self.input_bn(x_combined)\n                \n                # Initial transformation\n                x_hidden = self.initial_block(x_combined)\n                \n                # Residual connections\n                for block in self.residual_blocks:\n                    x_residual = block(x_hidden)\n                    x_hidden = x_hidden + x_residual\n                \n                # Output\n                output = self.output(x_hidden)\n                \n                if return_ae_loss:\n                    return output, decoded, x\n                elif return_encodings:\n                    return output, encoded\n                return output\n        \n        return CryptoMLPWithAutoEncoder()\n\n# =============================================================================\n# TRAINER CLASS\n# =============================================================================\n\nclass AutoEncoderTrainer:\n    \"\"\"Training logic for AutoEncoder models with enhanced tracking\"\"\"\n    \n    def __init__(self, config: AutoEncoderSimpleConfiguration, nn_builder: AutoEncoderNeuralNetwork):\n        self.config = config\n        self.nn_builder = nn_builder\n        self.training_history = {}\n    \n    def train_fold(self, train_df: pd.DataFrame, val_df: pd.DataFrame,\n                   test_df: pd.DataFrame, fold_idx: int) -> Dict[str, Any]:\n        \"\"\"Train a single fold of the AutoEncoder model with comprehensive tracking\"\"\"\n        \n        torch = self.nn_builder.torch\n        nn = self.nn_builder.nn\n        device = self.nn_builder.device\n        \n        # Extract features and labels\n        feature_cols = [col for col in train_df.columns if col not in ['label', 'original_index']]\n        \n        X_train = train_df[feature_cols].values\n        y_train = train_df['label'].values\n        train_indices = train_df['original_index'].values\n        \n        X_val = val_df[feature_cols].values\n        y_val = val_df['label'].values\n        val_indices = val_df['original_index'].values\n        \n        X_test = test_df[feature_cols].values\n        \n        # Handle invalid values\n        X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n        X_val = np.nan_to_num(X_val, nan=0.0, posinf=0.0, neginf=0.0)\n        X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n        \n        # Create data loaders\n        from torch.utils.data import Dataset, DataLoader\n        \n        class CryptoDataset(Dataset):\n            def __init__(self, features: np.ndarray, labels: Optional[np.ndarray] = None, indices: Optional[np.ndarray] = None):\n                self.features = torch.FloatTensor(features)\n                self.labels = torch.FloatTensor(labels) if labels is not None else None\n                self.indices = indices\n                \n            def __len__(self):\n                return len(self.features)\n                \n            def __getitem__(self, idx):\n                if self.labels is not None:\n                    return self.features[idx], self.labels[idx], idx\n                return self.features[idx], idx\n        \n        train_dataset = CryptoDataset(X_train, y_train, train_indices)\n        val_dataset = CryptoDataset(X_val, y_val, val_indices)\n        test_dataset = CryptoDataset(X_test, indices=np.arange(len(X_test)))\n        \n        train_loader = DataLoader(train_dataset, batch_size=self.config.batch_size, \n                                shuffle=True, num_workers=0)\n        val_loader = DataLoader(val_dataset, batch_size=self.config.batch_size, \n                              shuffle=False, num_workers=0)\n        test_loader = DataLoader(test_dataset, batch_size=self.config.batch_size, \n                               shuffle=False, num_workers=0)\n        \n        # Create model\n        num_features = X_train.shape[1]\n        model = self.nn_builder.create_full_model(num_features).to(device)\n        \n        # Initialize weights\n        def init_weights(m):\n            if isinstance(m, nn.Linear):\n                torch.nn.init.xavier_uniform_(m.weight)\n                m.bias.data.fill_(0.01)\n        \n        model.apply(init_weights)\n        \n        # Loss function\n        class CombinedLoss(nn.Module):\n            def __init__(self):\n                super().__init__()\n                self.mse = nn.MSELoss()\n                \n            def forward(self, y_pred, y_true, decoded=None, original=None):\n                # Main prediction loss\n                mse_loss = self.mse(y_pred, y_true)\n                \n                # Correlation loss\n                y_pred_centered = y_pred - y_pred.mean()\n                y_true_centered = y_true - y_true.mean()\n                \n                correlation = torch.sum(y_pred_centered * y_true_centered) / (\n                    torch.sqrt(torch.sum(y_pred_centered ** 2)) * \n                    torch.sqrt(torch.sum(y_true_centered ** 2)) + 1e-8\n                )\n                \n                # Autoencoder reconstruction loss\n                ae_loss = self.mse(decoded, original) if decoded is not None else 0\n                \n                total_loss = (self.config.mse_weight * mse_loss - \n                            self.config.corr_weight * correlation + \n                            self.config.ae_weight * ae_loss)\n                \n                return total_loss, correlation.item(), mse_loss.item(), ae_loss\n        \n        criterion = CombinedLoss()\n        optimizer = torch.optim.Adam(model.parameters(), lr=self.config.learning_rate,\n                                   weight_decay=self.config.weight_decay)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max',\n                                                              patience=5, factor=0.5,\n                                                              min_lr=self.config.min_lr)\n        \n        # Training history tracking\n        fold_history = {\n            'train_loss': [],\n            'val_loss': [],\n            'train_corr': [],\n            'val_corr': [],\n            'learning_rates': [],\n            'best_epoch': 0\n        }\n        \n        # Training loop\n        best_val_corr = -float('inf')\n        best_model_state = None\n        early_stop_counter = 0\n        \n        # Storage for validation predictions and encodings\n        best_val_predictions = None\n        best_val_encodings = None\n        \n        for epoch in range(self.config.num_epochs):\n            # Training\n            model.train()\n            train_losses = []\n            train_corrs = []\n            \n            for batch_features, batch_labels, _ in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n                batch_features = batch_features.to(device)\n                batch_labels = batch_labels.to(device)\n                \n                optimizer.zero_grad()\n                \n                outputs, decoded, original = model(batch_features, return_ae_loss=True)\n                loss, corr, mse_loss, ae_loss = criterion(outputs, batch_labels, decoded, original)\n                \n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n                optimizer.step()\n                \n                train_losses.append(loss.item())\n                train_corrs.append(corr)\n            \n            # Validation\n            model.eval()\n            val_predictions = []\n            val_targets = []\n            val_encodings = []\n            val_losses = []\n            val_corrs = []\n            \n            with torch.no_grad():\n                for batch_features, batch_labels, _ in val_loader:\n                    batch_features = batch_features.to(device)\n                    outputs, encoded = model(batch_features, return_encodings=True)\n                    \n                    val_predictions.extend(outputs.cpu().numpy().flatten())\n                    val_targets.extend(batch_labels.numpy().flatten())\n                    val_encodings.append(encoded.cpu().numpy())\n                    \n                    # Calculate validation loss\n                    outputs_loss, decoded, original = model(batch_features, return_ae_loss=True)\n                    loss, corr, _, _ = criterion(outputs_loss, batch_labels.to(device), decoded, original)\n                    val_losses.append(loss.item())\n                    val_corrs.append(corr)\n            \n            # Aggregate validation encodings\n            val_encodings = np.vstack(val_encodings)\n            \n            # Calculate metrics\n            val_corr = pearsonr(val_predictions, val_targets)[0]\n            avg_train_loss = np.mean(train_losses)\n            avg_val_loss = np.mean(val_losses)\n            avg_train_corr = np.mean(train_corrs)\n            \n            # Update learning rate\n            scheduler.step(val_corr)\n            current_lr = optimizer.param_groups[0]['lr']\n            \n            # Store history\n            fold_history['train_loss'].append(avg_train_loss)\n            fold_history['val_loss'].append(avg_val_loss)\n            fold_history['train_corr'].append(avg_train_corr)\n            fold_history['val_corr'].append(val_corr)\n            fold_history['learning_rates'].append(current_lr)\n            \n            # Early stopping check\n            if val_corr > best_val_corr:\n                best_val_corr = val_corr\n                best_model_state = model.state_dict().copy()\n                best_val_predictions = np.array(val_predictions)\n                best_val_encodings = val_encodings\n                early_stop_counter = 0\n                fold_history['best_epoch'] = epoch\n            else:\n                early_stop_counter += 1\n            \n            if (epoch + 1) % 10 == 0:\n                print(f\"  Epoch {epoch+1}/{self.config.num_epochs} | Val Corr: {val_corr:.6f} | LR: {current_lr:.6f}\")\n            \n            if early_stop_counter >= self.config.patience:\n                print(f\"  Early stopping at epoch {epoch+1}\")\n                break\n        \n        # Load best model and make predictions\n        model.load_state_dict(best_model_state)\n        model.eval()\n        \n        # Get test predictions and encodings\n        test_predictions = []\n        test_encodings = []\n        \n        with torch.no_grad():\n            for batch_features, _ in tqdm(test_loader, desc=\"Predicting\", leave=False):\n                batch_features = batch_features.to(device)\n                outputs, encoded = model(batch_features, return_encodings=True)\n                test_predictions.append(outputs.cpu().numpy())\n                test_encodings.append(encoded.cpu().numpy())\n        \n        test_predictions = np.vstack(test_predictions).flatten()\n        test_encodings = np.vstack(test_encodings)\n        \n        # Get training predictions and encodings for OOF\n        train_predictions = []\n        train_encodings = []\n        \n        model.eval()\n        with torch.no_grad():\n            train_loader_eval = DataLoader(train_dataset, batch_size=self.config.batch_size, \n                                         shuffle=False, num_workers=0)\n            for batch_features, _, _ in train_loader_eval:\n                batch_features = batch_features.to(device)\n                outputs, encoded = model(batch_features, return_encodings=True)\n                train_predictions.append(outputs.cpu().numpy())\n                train_encodings.append(encoded.cpu().numpy())\n        \n        train_predictions = np.vstack(train_predictions).flatten()\n        train_encodings = np.vstack(train_encodings)\n        \n        # Save model state\n        model_path = os.path.join(self.config.intermediate_dir, f'fold_{fold_idx}_model.pt')\n        torch.save({\n            'model_state_dict': best_model_state,\n            'fold_idx': fold_idx,\n            'best_val_corr': best_val_corr,\n            'config': self.config.__dict__,\n            'architecture': {\n                'input_size': num_features,\n                'encoding_size': self.config.encoding_size,\n                'hidden_size': self.config.hidden_size,\n                'num_blocks': self.config.num_blocks\n            }\n        }, model_path)\n        \n        # Store fold history\n        self.training_history[f'fold_{fold_idx}'] = fold_history\n        \n        return {\n            'fold_idx': fold_idx,\n            'best_val_corr': best_val_corr,\n            'test_predictions': test_predictions,\n            'test_encodings': test_encodings,\n            'val_predictions': best_val_predictions,\n            'val_encodings': best_val_encodings,\n            'val_indices': val_indices,\n            'train_predictions': train_predictions,\n            'train_encodings': train_encodings,\n            'train_indices': train_indices,\n            'fold_history': fold_history,\n            'model_metadata': {\n                'best_epoch': fold_history['best_epoch'],\n                'total_epochs': len(fold_history['train_loss']),\n                'final_lr': fold_history['learning_rates'][-1],\n                'early_stopped': early_stop_counter >= self.config.patience\n            }\n        }\n\n# =============================================================================\n# MAIN PIPELINE CLASS\n# =============================================================================\n\nclass AutoEncoderSimplePipeline:\n    \"\"\"AutoEncoder Simple Pipeline - uses simple averaging for ensemble\"\"\"\n    \n    def __init__(self, config: Optional[AutoEncoderSimpleConfiguration] = None):\n        self.config = config or AutoEncoderSimpleConfiguration()\n        \n    def run(self) -> float:\n        \"\"\"Execute the AutoEncoder Simple pipeline\"\"\"\n        print(\"\\nStarting AutoEncoder Simple Pipeline\")\n        print(\"=\"*80)\n        \n        # Update status\n        global_config.update_model_status(self.config.model_name, 'running')\n        \n        try:\n            # Configure PyTorch environment\n            print(\"Configuring PyTorch environment...\")\n            TorchEnvironmentManager.configure_environment()\n            \n            # Load data\n            train_df, test_df, sample_submission = self.load_and_prepare_data()\n            \n            # Initialize neural network builder\n            nn_builder = AutoEncoderNeuralNetwork(self.config)\n            \n            if not nn_builder.initialize_pytorch():\n                print(\"PyTorch initialization failed, using fallback method\")\n                return self.run_fallback_pipeline(train_df, test_df, sample_submission)\n            \n            # Create trainer\n            trainer = AutoEncoderTrainer(self.config, nn_builder)\n            \n            # Time series cross-validation\n            tss = TimeSeriesSplit(\n                n_splits=self.config.n_splits,\n                max_train_size=self.config.max_train_size,\n                gap=self.config.gap\n            )\n            \n            # Initialize storage\n            all_results = []\n            all_test_predictions = []\n            oof_predictions = np.full(len(train_df), np.nan)\n            oof_encodings = np.full((len(train_df), self.config.encoding_size), np.nan)\n            fold_assignments = np.full(len(train_df), -1)\n            \n            # Train folds\n            for fold_idx, (train_idx, val_idx) in enumerate(tss.split(train_df)):\n                print(f\"\\nTraining Fold {fold_idx + 1}/{self.config.n_splits}\")\n                \n                RandomSeedManager.set_all_seeds(self.config.seed + fold_idx)\n                \n                # Split data\n                fold_train_df = train_df.iloc[train_idx]\n                fold_val_df = train_df.iloc[val_idx]\n                \n                print(f\"  Train samples: {len(fold_train_df):,}\")\n                print(f\"  Val samples: {len(fold_val_df):,}\")\n                \n                try:\n                    fold_results = trainer.train_fold(fold_train_df, fold_val_df, test_df, fold_idx)\n                    all_results.append(fold_results)\n                    all_test_predictions.append(fold_results['test_predictions'])\n                    \n                    # Store OOF predictions\n                    oof_predictions[fold_results['val_indices']] = fold_results['val_predictions']\n                    oof_encodings[fold_results['val_indices']] = fold_results['val_encodings']\n                    fold_assignments[fold_results['val_indices']] = fold_idx\n                    \n                    print(f\"  Fold {fold_idx + 1} completed | Best Corr: {fold_results['best_val_corr']:.6f}\")\n                    \n                except Exception as e:\n                    print(f\"  Error in fold {fold_idx + 1}: {e}\")\n                    print(\"  Continuing with remaining folds...\")\n                \n                # Clean up memory\n                aggressive_memory_cleanup()\n            \n            if not all_results:\n                print(\"All folds failed, using fallback method\")\n                return self.run_fallback_pipeline(train_df, test_df, sample_submission)\n            \n            # Create SIMPLE ensemble predictions (key difference from weighted version)\n            print(\"\\nCreating simple average ensemble predictions...\")\n            ensemble_predictions = np.mean(all_test_predictions, axis=0)\n            test_encodings_ensemble = np.mean([r['test_encodings'] for r in all_results], axis=0)\n            \n            # Calculate score\n            correlations = [r['best_val_corr'] for r in all_results]\n            final_score = np.mean(correlations)\n            \n            # Save all outputs\n            self.save_outputs(\n                ensemble_predictions=ensemble_predictions,\n                oof_predictions=oof_predictions,\n                oof_encodings=oof_encodings,\n                test_encodings=test_encodings_ensemble,\n                train_df=train_df,\n                sample_submission=sample_submission,\n                all_results=all_results,\n                final_score=final_score,\n                fold_assignments=fold_assignments,\n                trainer=trainer\n            )\n            \n            # Update status\n            global_config.update_model_status(self.config.model_name, 'completed', score=final_score)\n            \n            print(f\"\\n✅ AutoEncoder Simple pipeline completed successfully\")\n            print(f\"  Final Score: {final_score:.6f}\")\n            \n            return final_score\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n            print(f\"\\n❌ AutoEncoder Simple pipeline failed: {error_msg}\")\n            raise\n            \n        finally:\n            aggressive_memory_cleanup()\n            print(\"\\n\" + \"=\"*80)\n            print(\"Current Execution Status:\")\n            print(global_config.get_execution_summary())\n            print(\"=\"*80)\n    \n    def load_and_prepare_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n        \"\"\"Load and prepare data\"\"\"\n        print(\"Loading data...\")\n        train_df = pd.read_parquet(global_config.train_path)\n        test_df = pd.read_parquet(global_config.test_path)\n        sample_submission = pd.read_csv(global_config.sample_sub_path)\n        \n        # Add original indices\n        train_df['original_index'] = range(len(train_df))\n        test_df['original_index'] = range(len(test_df))\n        \n        # Select features\n        available_features = [col for col in self.config.feature_columns if col in train_df.columns]\n        self.config.actual_features = available_features\n        \n        train_df = train_df[available_features + ['label', 'original_index']]\n        test_df = test_df[available_features + ['original_index']]\n        \n        # Optimize memory\n        train_df = reduce_memory_usage(train_df, verbose=False)\n        test_df = reduce_memory_usage(test_df, verbose=False)\n        \n        print(f\"Data loaded - Train: {train_df.shape}, Test: {test_df.shape}\")\n        return train_df, test_df, sample_submission\n    \n    def run_fallback_pipeline(self, train_df: pd.DataFrame, test_df: pd.DataFrame, \n                            sample_submission: pd.DataFrame) -> float:\n        \"\"\"Fallback pipeline using Ridge regression\"\"\"\n        print(\"\\nRunning fallback Ridge regression pipeline...\")\n        \n        feature_cols = [col for col in train_df.columns if col not in ['label', 'original_index']]\n        X_train = train_df[feature_cols].values\n        y_train = train_df['label'].values\n        X_test = test_df[feature_cols].values\n        \n        # Handle invalid values\n        X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n        X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n        \n        # Scale features\n        scaler = StandardScaler()\n        X_train_scaled = scaler.fit_transform(X_train)\n        X_test_scaled = scaler.transform(X_test)\n        \n        # Cross-validation for OOF\n        kf = KFold(n_splits=5, shuffle=True, random_state=self.config.seed)\n        oof_predictions = np.zeros(len(X_train))\n        test_predictions = []\n        \n        for train_idx, val_idx in kf.split(X_train_scaled):\n            X_fold_train, X_fold_val = X_train_scaled[train_idx], X_train_scaled[val_idx]\n            y_fold_train = y_train[train_idx]\n            \n            model = Ridge(alpha=1.0, random_state=self.config.seed)\n            model.fit(X_fold_train, y_fold_train)\n            \n            oof_predictions[val_idx] = model.predict(X_fold_val)\n            test_predictions.append(model.predict(X_test_scaled))\n        \n        # Average test predictions\n        test_predictions = np.mean(test_predictions, axis=0)\n        \n        # Calculate score\n        oof_score = pearsonr(oof_predictions, y_train)[0]\n        \n        # Create dummy encodings\n        dummy_encodings = np.zeros((len(X_train), self.config.encoding_size))\n        test_dummy_encodings = np.zeros((len(X_test), self.config.encoding_size))\n        \n        # Save outputs\n        self.save_outputs(\n            ensemble_predictions=test_predictions,\n            oof_predictions=oof_predictions,\n            oof_encodings=dummy_encodings,\n            test_encodings=test_dummy_encodings,\n            train_df=train_df,\n            sample_submission=sample_submission,\n            all_results=[],\n            final_score=oof_score,\n            fold_assignments=np.zeros(len(train_df)),\n            trainer=None,\n            is_fallback=True\n        )\n        \n        global_config.update_model_status(self.config.model_name, 'completed', score=oof_score)\n        return oof_score\n    \n    def save_outputs(self, ensemble_predictions, oof_predictions, oof_encodings, test_encodings,\n                    train_df, sample_submission, all_results, final_score, fold_assignments,\n                    trainer, is_fallback=False):\n        \"\"\"Save all outputs for the simple ensemble\"\"\"\n        \n        # Save submission\n        submission = sample_submission.copy()\n        submission.iloc[:, 1] = ensemble_predictions\n        submission.to_csv(self.config.submission_path, index=False)\n        \n        # Register submission\n        global_config.register_model_output(\n            self.config.model_name,\n            'submission',\n            self.config.submission_path,\n            'submission',\n            metadata={\n                'method': 'simple_average' if not is_fallback else 'ridge_fallback',\n                'score': float(final_score),\n                'n_folds': len(all_results) if all_results else 5\n            }\n        )\n        \n        # Save OOF predictions\n        oof_df = pd.DataFrame({\n            'row_id': train_df['original_index'].values,\n            'true_label': train_df['label'].values,\n            'prediction': oof_predictions,\n            'fold': fold_assignments\n        })\n        \n        # Add encoded features\n        for i in range(oof_encodings.shape[1]):\n            oof_df[f'encoded_{i}'] = oof_encodings[:, i]\n        \n        oof_df.to_csv(self.config.oof_predictions_path, index=False)\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'oof_predictions',\n            self.config.oof_predictions_path,\n            'oof_predictions',\n            metadata={'n_samples': len(oof_df), 'encoding_dims': oof_encodings.shape[1]}\n        )\n        \n        # Save encoded features\n        encoding_df = pd.DataFrame(\n            oof_encodings,\n            columns=[f'encoded_{i}' for i in range(oof_encodings.shape[1])]\n        )\n        encoding_df['row_id'] = train_df['original_index'].values\n        \n        cols = ['row_id'] + [col for col in encoding_df.columns if col != 'row_id']\n        encoding_df = encoding_df[cols]\n        \n        encoding_df.to_csv(self.config.encoded_features_path, index=False)\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'encoded_features',\n            self.config.encoded_features_path,\n            'features',\n            metadata={'encoding_dims': oof_encodings.shape[1]}\n        )\n        \n        # Save meta-features\n        meta_features = self.create_meta_features(oof_predictions, oof_encodings, train_df['label'].values)\n        meta_features['row_id'] = train_df['original_index'].values\n        \n        cols = ['row_id'] + [col for col in meta_features.columns if col != 'row_id']\n        meta_features = meta_features[cols]\n        \n        meta_features.to_csv(self.config.meta_features_path, index=False)\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'meta_features',\n            self.config.meta_features_path,\n            'features',\n            metadata={'n_features': len(meta_features.columns) - 1}\n        )\n        \n        # Save configuration\n        self.config.save()\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'config',\n            self.config.config_path,\n            'config',\n            metadata={'encoding_size': self.config.encoding_size, 'hidden_size': self.config.hidden_size}\n        )\n        \n        print(f\"\\nAll outputs saved for {self.config.model_name}\")\n    \n    def create_meta_features(self, predictions, encodings, y_true):\n        \"\"\"Create meta-features for the simple ensemble\"\"\"\n        meta_features = {}\n        \n        # Basic features\n        meta_features['prediction'] = predictions\n        meta_features['error'] = predictions - y_true\n        meta_features['abs_error'] = np.abs(predictions - y_true)\n        meta_features['squared_error'] = (predictions - y_true) ** 2\n        \n        # Encoding statistics\n        meta_features['encoding_mean'] = np.mean(encodings, axis=1)\n        meta_features['encoding_std'] = np.std(encodings, axis=1)\n        meta_features['encoding_min'] = np.min(encodings, axis=1)\n        meta_features['encoding_max'] = np.max(encodings, axis=1)\n        meta_features['encoding_l1_norm'] = np.sum(np.abs(encodings), axis=1)\n        meta_features['encoding_l2_norm'] = np.sqrt(np.sum(encodings ** 2, axis=1))\n        \n        return pd.DataFrame(meta_features)\n\n# =============================================================================\n# MAIN EXECUTION\n# =============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\\n🧠 Running AutoEncoder Simple Pipeline\")\n    print(\"-\"*80)\n    \n    # Clean memory\n    aggressive_memory_cleanup()\n    \n    # Create configuration\n    config = AutoEncoderSimpleConfiguration(\n        num_epochs=80,\n        encoding_size=128,\n        hidden_size=256,\n        num_blocks=8,\n        dropout=0.7,\n        noise_std=0.05\n    )\n    \n    # Create and run pipeline\n    pipeline = AutoEncoderSimplePipeline(config)\n    final_score = pipeline.run()","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# AUTOENCODER WEIGHTED PIPELINE - STANDALONE IMPLEMENTATION\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - AutoEncoder Weighted Pipeline\nWeighted averaging ensemble version with validation-based weights\n\"\"\"\n\nimport subprocess\nimport sys\nimport os\nimport gc\nimport warnings\nimport json\nimport random\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr\nfrom typing import List, Dict, Tuple, Optional, Any\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nfrom sklearn.model_selection import TimeSeriesSplit, KFold\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings('ignore')\n\n# Install required packages\nprint(\"Installing packages for AutoEncoder Weighted pipeline...\")\npackages_to_install = [\n    'torch==2.0.1',\n    'torchvision==0.15.2',\n    'tqdm==4.65.0'\n]\n\nprint(\"Installing PyTorch (CPU version)...\")\nsubprocess.check_call([\n    sys.executable, \"-m\", \"pip\", \"install\", \n    \"torch==2.0.1\", \"torchvision==0.15.2\",\n    \"--index-url\", \"https://download.pytorch.org/whl/cpu\",\n    \"--quiet\"\n])\n\nsubprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"tqdm==4.65.0\", \"--quiet\"])\n\n@dataclass\nclass AutoEncoderWeightedConfiguration:\n    \"\"\"Configuration for AutoEncoder Weighted Pipeline\"\"\"\n    \n    # Model identification\n    model_name: str = \"autoencoder_weighted\"\n    model_directory: str = \"\"\n    \n    # Model-specific features\n    feature_columns: List[str] = field(default_factory=list)\n    actual_features: List[str] = field(default_factory=list)\n    \n    # AutoEncoder architecture parameters\n    encoding_size: int = 128\n    hidden_size: int = 256\n    dropout: float = 0.7\n    ae_dropout: float = 0.3\n    noise_std: float = 0.05\n    num_blocks: int = 8\n    \n    # Training parameters\n    num_epochs: int = 80\n    batch_size: int = 4096\n    learning_rate: float = 0.0001\n    weight_decay: float = 5e-3\n    patience: int = 10\n    min_lr: float = 1e-6\n    \n    # Loss weights\n    mse_weight: float = 0.25\n    corr_weight: float = 0.6\n    ae_weight: float = 0.15\n    \n    # Cross-validation parameters\n    n_splits: int = 5\n    max_train_size: int = 100_000_000\n    gap: int = 1\n    \n    # Random seed\n    seed: int = 42\n    \n    # Weighted ensemble specific\n    weight_power: float = 2.0  # Power to raise correlation scores for weighting\n    min_weight: float = 0.0    # Minimum weight for any fold\n    \n    def __post_init__(self):\n        \"\"\"Initialize configuration and register model\"\"\"\n        if not self.model_directory:\n            self.model_directory = os.path.join(global_config.base_dir, \"autoencoder_weighted\")\n        \n        # Register model with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # Set default features if not provided\n        if not self.feature_columns:\n            self.feature_columns = [\n                \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n                \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n                \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\",\n                \"X188\", \"X207\", \"X219\", \"X233\", \"X245\"\n            ]\n        \n        # Output paths\n        self.intermediate_dir = os.path.join(self.model_directory, \"fold_models\")\n        self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n        self.config_path = os.path.join(self.model_directory, \"config.json\")\n        self.performance_metrics_path = os.path.join(self.model_directory, \"performance_metrics.json\")\n        self.weights_path = os.path.join(self.model_directory, \"fold_weights.json\")\n        \n        # Meta-learning data paths\n        self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n        self.encoded_features_path = os.path.join(self.model_directory, \"encoded_features.csv\")\n        self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n        self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n        self.training_history_path = os.path.join(self.model_directory, \"training_history.json\")\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n        Path(self.intermediate_dir).mkdir(parents=True, exist_ok=True)\n    \n    def save(self):\n        \"\"\"Save configuration to JSON file\"\"\"\n        config_dict = {k: v for k, v in self.__dict__.items() \n                      if not k.startswith('_') and k not in ['model_directory', 'intermediate_dir']}\n        with open(self.config_path, 'w') as f:\n            json.dump(config_dict, f, indent=2)\n\nclass AutoEncoderWeightedPipeline:\n    \"\"\"AutoEncoder Weighted Pipeline - uses weighted averaging based on validation scores\"\"\"\n    \n    def __init__(self, config: Optional[AutoEncoderWeightedConfiguration] = None):\n        self.config = config or AutoEncoderWeightedConfiguration()\n        \n    def run(self) -> float:\n        \"\"\"Execute the AutoEncoder Weighted pipeline\"\"\"\n        print(\"\\nStarting AutoEncoder Weighted Pipeline\")\n        print(\"=\"*80)\n        \n        # Update status\n        global_config.update_model_status(self.config.model_name, 'running')\n        \n        try:\n            # Configure PyTorch environment\n            print(\"Configuring PyTorch environment...\")\n            TorchEnvironmentManager.configure_environment()\n            \n            # Load data\n            train_df, test_df, sample_submission = self.load_and_prepare_data()\n            \n            # Initialize neural network builder\n            nn_builder = AutoEncoderNeuralNetwork(self.config)\n            \n            if not nn_builder.initialize_pytorch():\n                print(\"PyTorch initialization failed, using fallback method\")\n                return self.run_fallback_pipeline(train_df, test_df, sample_submission)\n            \n            # Create trainer\n            trainer = AutoEncoderTrainer(self.config, nn_builder)\n            \n            # Time series cross-validation\n            tss = TimeSeriesSplit(\n                n_splits=self.config.n_splits,\n                max_train_size=self.config.max_train_size,\n                gap=self.config.gap\n            )\n            \n            # Initialize storage\n            all_results = []\n            all_test_predictions = []\n            oof_predictions = np.full(len(train_df), np.nan)\n            oof_encodings = np.full((len(train_df), self.config.encoding_size), np.nan)\n            fold_assignments = np.full(len(train_df), -1)\n            \n            # Train folds\n            for fold_idx, (train_idx, val_idx) in enumerate(tss.split(train_df)):\n                print(f\"\\nTraining Fold {fold_idx + 1}/{self.config.n_splits}\")\n                \n                RandomSeedManager.set_all_seeds(self.config.seed + fold_idx)\n                \n                # Split data\n                fold_train_df = train_df.iloc[train_idx]\n                fold_val_df = train_df.iloc[val_idx]\n                \n                print(f\"  Train samples: {len(fold_train_df):,}\")\n                print(f\"  Val samples: {len(fold_val_df):,}\")\n                \n                try:\n                    fold_results = trainer.train_fold(fold_train_df, fold_val_df, test_df, fold_idx)\n                    all_results.append(fold_results)\n                    all_test_predictions.append(fold_results['test_predictions'])\n                    \n                    # Store OOF predictions\n                    oof_predictions[fold_results['val_indices']] = fold_results['val_predictions']\n                    oof_encodings[fold_results['val_indices']] = fold_results['val_encodings']\n                    fold_assignments[fold_results['val_indices']] = fold_idx\n                    \n                    print(f\"  Fold {fold_idx + 1} completed | Best Corr: {fold_results['best_val_corr']:.6f}\")\n                    \n                except Exception as e:\n                    print(f\"  Error in fold {fold_idx + 1}: {e}\")\n                    print(\"  Continuing with remaining folds...\")\n                \n                # Clean up memory\n                aggressive_memory_cleanup()\n            \n            if not all_results:\n                print(\"All folds failed, using fallback method\")\n                return self.run_fallback_pipeline(train_df, test_df, sample_submission)\n            \n            # Create WEIGHTED ensemble predictions (key difference from simple version)\n            print(\"\\nCreating weighted ensemble predictions...\")\n            \n            # Calculate weights based on validation scores\n            val_scores = np.array([r['best_val_corr'] for r in all_results])\n            \n            # Apply power transformation to emphasize better models\n            weights = np.power(np.maximum(val_scores, self.config.min_weight), self.config.weight_power)\n            weights = weights / weights.sum()\n            \n            print(\"Fold weights based on validation scores:\")\n            for i, (score, weight) in enumerate(zip(val_scores, weights)):\n                print(f\"  Fold {i+1}: score={score:.4f}, weight={weight:.4f}\")\n            \n            # Create weighted ensemble\n            ensemble_predictions = np.average(all_test_predictions, axis=0, weights=weights)\n            test_encodings_ensemble = np.average(\n                [r['test_encodings'] for r in all_results], \n                axis=0, \n                weights=weights\n            )\n            \n            # Calculate final score\n            final_score = np.average(val_scores, weights=weights)\n            \n            # Save weights information\n            weights_info = {\n                'fold_scores': val_scores.tolist(),\n                'fold_weights': weights.tolist(),\n                'weight_power': self.config.weight_power,\n                'min_weight': self.config.min_weight\n            }\n            \n            with open(self.config.weights_path, 'w') as f:\n                json.dump(weights_info, f, indent=2)\n            \n            # Save all outputs\n            self.save_outputs(\n                ensemble_predictions=ensemble_predictions,\n                oof_predictions=oof_predictions,\n                oof_encodings=oof_encodings,\n                test_encodings=test_encodings_ensemble,\n                train_df=train_df,\n                sample_submission=sample_submission,\n                all_results=all_results,\n                final_score=final_score,\n                fold_assignments=fold_assignments,\n                trainer=trainer,\n                weights=weights\n            )\n            \n            # Update status\n            global_config.update_model_status(self.config.model_name, 'completed', score=final_score)\n            \n            print(f\"\\n✅ AutoEncoder Weighted pipeline completed successfully\")\n            print(f\"  Final Weighted Score: {final_score:.6f}\")\n            \n            return final_score\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n            print(f\"\\n❌ AutoEncoder Weighted pipeline failed: {error_msg}\")\n            raise\n            \n        finally:\n            aggressive_memory_cleanup()\n            print(\"\\n\" + \"=\"*80)\n            print(\"Current Execution Status:\")\n            print(global_config.get_execution_summary())\n            print(\"=\"*80)\n    \n    def load_and_prepare_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n        \"\"\"Load and prepare data\"\"\"\n        print(\"Loading data...\")\n        train_df = pd.read_parquet(global_config.train_path)\n        test_df = pd.read_parquet(global_config.test_path)\n        sample_submission = pd.read_csv(global_config.sample_sub_path)\n        \n        # Add original indices\n        train_df['original_index'] = range(len(train_df))\n        test_df['original_index'] = range(len(test_df))\n        \n        # Select features\n        available_features = [col for col in self.config.feature_columns if col in train_df.columns]\n        self.config.actual_features = available_features\n        \n        train_df = train_df[available_features + ['label', 'original_index']]\n        test_df = test_df[available_features + ['original_index']]\n        \n        # Optimize memory\n        train_df = reduce_memory_usage(train_df, verbose=False)\n        test_df = reduce_memory_usage(test_df, verbose=False)\n        \n        print(f\"Data loaded - Train: {train_df.shape}, Test: {test_df.shape}\")\n        return train_df, test_df, sample_submission\n    \n    def run_fallback_pipeline(self, train_df: pd.DataFrame, test_df: pd.DataFrame, \n                            sample_submission: pd.DataFrame) -> float:\n        \"\"\"Fallback pipeline using Ridge regression\"\"\"\n        print(\"\\nRunning fallback Ridge regression pipeline...\")\n        \n        feature_cols = [col for col in train_df.columns if col not in ['label', 'original_index']]\n        X_train = train_df[feature_cols].values\n        y_train = train_df['label'].values\n        X_test = test_df[feature_cols].values\n        \n        # Handle invalid values\n        X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n        X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n        \n        # Scale features\n        scaler = StandardScaler()\n        X_train_scaled = scaler.fit_transform(X_train)\n        X_test_scaled = scaler.transform(X_test)\n        \n        # Cross-validation for OOF with weighted ensemble\n        kf = KFold(n_splits=5, shuffle=True, random_state=self.config.seed)\n        oof_predictions = np.zeros(len(X_train))\n        test_predictions = []\n        fold_scores = []\n        \n        for train_idx, val_idx in kf.split(X_train_scaled):\n            X_fold_train, X_fold_val = X_train_scaled[train_idx], X_train_scaled[val_idx]\n            y_fold_train, y_fold_val = y_train[train_idx], y_train[val_idx]\n            \n            model = Ridge(alpha=1.0, random_state=self.config.seed)\n            model.fit(X_fold_train, y_fold_train)\n            \n            val_pred = model.predict(X_fold_val)\n            oof_predictions[val_idx] = val_pred\n            \n            # Calculate fold score\n            fold_score = pearsonr(val_pred, y_fold_val)[0]\n            fold_scores.append(fold_score)\n            \n            test_predictions.append(model.predict(X_test_scaled))\n        \n        # Weighted average test predictions\n        fold_scores = np.array(fold_scores)\n        weights = np.power(np.maximum(fold_scores, 0), self.config.weight_power)\n        weights = weights / weights.sum()\n        \n        test_predictions = np.average(test_predictions, axis=0, weights=weights)\n        \n        # Calculate weighted score\n        oof_score = np.average(fold_scores, weights=weights)\n        \n        # Create dummy encodings\n        dummy_encodings = np.zeros((len(X_train), self.config.encoding_size))\n        test_dummy_encodings = np.zeros((len(X_test), self.config.encoding_size))\n        \n        # Save outputs\n        self.save_outputs(\n            ensemble_predictions=test_predictions,\n            oof_predictions=oof_predictions,\n            oof_encodings=dummy_encodings,\n            test_encodings=test_dummy_encodings,\n            train_df=train_df,\n            sample_submission=sample_submission,\n            all_results=[],\n            final_score=oof_score,\n            fold_assignments=np.zeros(len(train_df)),\n            trainer=None,\n            weights=weights,\n            is_fallback=True\n        )\n        \n        global_config.update_model_status(self.config.model_name, 'completed', score=oof_score)\n        return oof_score\n    \n    def save_outputs(self, ensemble_predictions, oof_predictions, oof_encodings, test_encodings,\n                    train_df, sample_submission, all_results, final_score, fold_assignments,\n                    trainer, weights, is_fallback=False):\n        \"\"\"Save all outputs for the weighted ensemble\"\"\"\n        \n        # Save submission\n        submission = sample_submission.copy()\n        submission.iloc[:, 1] = ensemble_predictions\n        submission.to_csv(self.config.submission_path, index=False)\n        \n        # Register submission\n        global_config.register_model_output(\n            self.config.model_name,\n            'submission',\n            self.config.submission_path,\n            'submission',\n            metadata={\n                'method': 'weighted_average' if not is_fallback else 'ridge_fallback_weighted',\n                'score': float(final_score),\n                'n_folds': len(all_results) if all_results else 5,\n                'weights': weights.tolist()\n            }\n        )\n        \n        # Save OOF predictions\n        oof_df = pd.DataFrame({\n            'row_id': train_df['original_index'].values,\n            'true_label': train_df['label'].values,\n            'prediction': oof_predictions,\n            'fold': fold_assignments\n        })\n        \n        # Add encoded features\n        for i in range(oof_encodings.shape[1]):\n            oof_df[f'encoded_{i}'] = oof_encodings[:, i]\n        \n        # Add fold weights as metadata\n        oof_df['fold_weight'] = [weights[int(f)] if f >= 0 else 0 for f in fold_assignments]\n        \n        oof_df.to_csv(self.config.oof_predictions_path, index=False)\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'oof_predictions',\n            self.config.oof_predictions_path,\n            'oof_predictions',\n            metadata={\n                'n_samples': len(oof_df), \n                'encoding_dims': oof_encodings.shape[1],\n                'fold_weights': weights.tolist()\n            }\n        )\n        \n        # Save encoded features\n        encoding_df = pd.DataFrame(\n            oof_encodings,\n            columns=[f'encoded_{i}' for i in range(oof_encodings.shape[1])]\n        )\n        encoding_df['row_id'] = train_df['original_index'].values\n        \n        cols = ['row_id'] + [col for col in encoding_df.columns if col != 'row_id']\n        encoding_df = encoding_df[cols]\n        \n        encoding_df.to_csv(self.config.encoded_features_path, index=False)\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'encoded_features',\n            self.config.encoded_features_path,\n            'features',\n            metadata={'encoding_dims': oof_encodings.shape[1]}\n        )\n        \n        # Save meta-features\n        meta_features = self.create_meta_features(\n            oof_predictions, oof_encodings, train_df['label'].values, weights, fold_assignments\n        )\n        meta_features['row_id'] = train_df['original_index'].values\n        \n        cols = ['row_id'] + [col for col in meta_features.columns if col != 'row_id']\n        meta_features = meta_features[cols]\n        \n        meta_features.to_csv(self.config.meta_features_path, index=False)\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'meta_features',\n            self.config.meta_features_path,\n            'features',\n            metadata={'n_features': len(meta_features.columns) - 1}\n        )\n        \n        # Save configuration\n        self.config.save()\n        \n        global_config.register_model_output(\n            self.config.model_name,\n            'config',\n            self.config.config_path,\n            'config',\n            metadata={\n                'encoding_size': self.config.encoding_size, \n                'hidden_size': self.config.hidden_size,\n                'weight_power': self.config.weight_power\n            }\n        )\n        \n        # Register weights file\n        global_config.register_model_output(\n            self.config.model_name,\n            'fold_weights',\n            self.config.weights_path,\n            'analysis',\n            metadata={'weights': weights.tolist()}\n        )\n        \n        print(f\"\\nAll outputs saved for {self.config.model_name}\")\n    \n    def create_meta_features(self, predictions, encodings, y_true, weights, fold_assignments):\n        \"\"\"Create meta-features for the weighted ensemble\"\"\"\n        meta_features = {}\n        \n        # Basic features\n        meta_features['prediction'] = predictions\n        meta_features['error'] = predictions - y_true\n        meta_features['abs_error'] = np.abs(predictions - y_true)\n        meta_features['squared_error'] = (predictions - y_true) ** 2\n        \n        # Weight-related features\n        meta_features['fold_weight'] = [weights[int(f)] if f >= 0 else 0 for f in fold_assignments]\n        meta_features['weighted_prediction'] = predictions * meta_features['fold_weight']\n        \n        # Encoding statistics\n        meta_features['encoding_mean'] = np.mean(encodings, axis=1)\n        meta_features['encoding_std'] = np.std(encodings, axis=1)\n        meta_features['encoding_min'] = np.min(encodings, axis=1)\n        meta_features['encoding_max'] = np.max(encodings, axis=1)\n        meta_features['encoding_l1_norm'] = np.sum(np.abs(encodings), axis=1)\n        meta_features['encoding_l2_norm'] = np.sqrt(np.sum(encodings ** 2, axis=1))\n        \n        # Weighted encoding features\n        meta_features['weighted_encoding_mean'] = meta_features['encoding_mean'] * meta_features['fold_weight']\n        \n        return pd.DataFrame(meta_features)\n\n# Copy necessary classes from the original implementation\nclass TorchEnvironmentManager:\n    @staticmethod\n    def configure_environment():\n        os.environ['CUDA_VISIBLE_DEVICES'] = ''\n        os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:128'\n        os.environ['PYTORCH_NO_CUDA_MEMORY_CACHING'] = '1'\n        os.environ['TRITON_CACHE_DIR'] = f'/tmp/triton_cache_{os.getpid()}'\n        os.environ['DISABLE_TRITON'] = '1'\n\nclass RandomSeedManager:\n    @staticmethod\n    def set_all_seeds(seed: int):\n        random.seed(seed)\n        np.random.seed(seed)\n        os.environ['PYTHONHASHSEED'] = str(seed)\n        try:\n            import torch\n            torch.manual_seed(seed)\n            torch.cuda.manual_seed(seed)\n            torch.cuda.manual_seed_all(seed)\n            torch.backends.cudnn.deterministic = True\n            torch.backends.cudnn.benchmark = False\n        except:\n            pass\n\n# Include AutoEncoderNeuralNetwork and AutoEncoderTrainer classes here\n# (Same as in original but configured for weighted ensemble)\n\n# Main execution\nif __name__ == \"__main__\":\n    print(\"\\n🧠 Running AutoEncoder Weighted Pipeline\")\n    print(\"-\"*80)\n    \n    # Clean memory\n    aggressive_memory_cleanup()\n    \n    # Create configuration\n    config = AutoEncoderWeightedConfiguration(\n        num_epochs=80,\n        encoding_size=128,\n        hidden_size=256,\n        num_blocks=8,\n        dropout=0.7,\n        noise_std=0.05,\n        weight_power=2.0  # Emphasize better performing folds\n    )\n    \n    # Create and run pipeline\n    pipeline = AutoEncoderWeightedPipeline(config)\n    final_score = pipeline.run()","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # MARKET MICROSTRUCTURE XGBOOST PIPELINE IMPLEMENTATION - ENHANCED FOR META-LEARNING\n# # ===========================================================================================\n# # ===========================================================================================\n\n# #!/usr/bin/env python\n# # -*- coding: utf-8 -*-\n# \"\"\"\n# DRW Crypto Market Prediction - Market Microstructure XGBoost Pipeline\n# Enhanced version that saves comprehensive OOF predictions and metadata for meta-learning\n# \"\"\"\n\n# # ===========================================================================================\n# # PACKAGE INSTALLATION FOR MARKET MICROSTRUCTURE PIPELINE\n# # ===========================================================================================\n\n# import subprocess\n# import sys\n# import os\n# import gc\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # Check that setup cell was run\n# try:\n#     global_config\n#     print(\"✓ Global configuration found\")\n# except NameError:\n#     raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# # Install required packages for this pipeline\n# print(\"Installing packages for Market Microstructure XGBoost pipeline...\")\n# packages_to_install = [\n#     'xgboost==2.0.3',      # Specific version for stability\n#     'optuna==3.5.0',       # Hyperparameter optimization\n#     'pandas',\n#     'numpy',\n#     'scipy',\n#     'scikit-learn',\n#     'matplotlib'\n# ]\n\n# for package in packages_to_install:\n#     subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n\n# import pandas as pd\n# import numpy as np\n# from scipy.stats import pearsonr\n# from typing import List, Dict, Tuple, Optional, Any\n# from pathlib import Path\n# import json\n# from sklearn.model_selection import KFold\n# from sklearn.linear_model import Ridge\n# import matplotlib.pyplot as plt\n# from datetime import datetime\n\n# # Import packages after installation\n# import xgboost as xgb\n# from xgboost import XGBRegressor\n# import optuna\n\n# # Custom JSON encoder to handle numpy types\n# class NumpyEncoder(json.JSONEncoder):\n#     \"\"\"Custom encoder for numpy data types\"\"\"\n#     def default(self, obj):\n#         if isinstance(obj, (np.integer, np.int_, np.intc, np.intp, np.int8,\n#                           np.int16, np.int32, np.int64, np.uint8, np.uint16,\n#                           np.uint32, np.uint64)):\n#             return int(obj)\n#         elif isinstance(obj, (np.floating, np.float_, np.float16, np.float32, np.float64)):\n#             return float(obj)\n#         elif isinstance(obj, (np.complex_, np.complex64, np.complex128)):\n#             return {'real': obj.real, 'imag': obj.imag}\n#         elif isinstance(obj, np.ndarray):\n#             return obj.tolist()\n#         elif isinstance(obj, np.bool_):\n#             return bool(obj)\n#         elif isinstance(obj, np.void):\n#             return None\n#         return super(NumpyEncoder, self).default(obj)\n\n# class MarketMicrostructureConfiguration:\n#     \"\"\"Configuration for Market Microstructure XGBoost Pipeline with model-specific settings\"\"\"\n#     def __init__(self):\n#         # Model identification\n#         self.model_name = \"market_microstructure\"\n#         self.model_directory = os.path.join(global_config.base_dir, \"market_microstructure_xgboost\")\n        \n#         # Register model with global configuration\n#         global_config.register_model(self.model_name, self.model_directory)\n        \n#         # Data paths from global configuration\n#         self.train_path = global_config.train_path\n#         self.test_path = global_config.test_path\n#         self.sample_sub_path = global_config.sample_sub_path\n        \n#         # Model parameters\n#         self.target = \"label\"\n#         self.n_folds = 5\n#         self.seed = 42\n        \n#         # Optuna optimization settings\n#         self.run_optuna = True\n#         self.n_optuna_trials = 250\n        \n#         # XGBoost parameters (optimized for market microstructure features)\n#         self.xgb_params = {\n#             \"tree_method\": \"hist\",  # Changed from gpu_hist for better compatibility\n#             \"device\": \"cpu\",\n#             \"colsample_bylevel\": 0.7,\n#             \"colsample_bynode\": 0.7,\n#             \"colsample_bytree\": 0.7,\n#             \"gamma\": 1.5,\n#             \"learning_rate\": 0.02,\n#             \"max_depth\": 15,\n#             \"max_leaves\": 20,\n#             \"min_child_weight\": 10,\n#             \"n_estimators\": 1500,\n#             \"n_jobs\": -1,\n#             \"random_state\": 42,\n#             \"reg_alpha\": 30,\n#             \"reg_lambda\": 60,\n#             \"subsample\": 0.08,\n#             \"verbosity\": 0,\n#             \"eval_metric\": \"rmse\"\n#         }\n        \n#         # Output paths\n#         self.intermediate_dir = os.path.join(self.model_directory, \"intermediate_predictions\")\n#         self.submission_file = os.path.join(self.model_directory, \"submission.csv\")\n#         self.metrics_file = os.path.join(self.model_directory, \"model_metrics.csv\")\n#         self.feature_importance_file = os.path.join(self.model_directory, \"feature_importance.csv\")\n#         self.optuna_results_file = os.path.join(self.model_directory, \"optuna_results.json\")\n        \n#         # New paths for meta-learning data\n#         self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n#         self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n#         self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n#         self.row_level_metrics_path = os.path.join(self.model_directory, \"row_level_metrics.csv\")\n#         self.ridge_ensemble_oof_path = os.path.join(self.model_directory, \"ridge_ensemble_oof.csv\")\n        \n#         # Model-specific columns to drop (these are typically redundant or problematic features)\n#         self.cols_to_drop = [\n#             'X697', 'X698', 'X699', 'X700', 'X701', 'X702', 'X703', 'X704', 'X705', 'X706', \n#             'X707', 'X708', 'X709', 'X710', 'X711', 'X712', 'X713', 'X714', 'X715', 'X716',\n#             'X717', 'X864', 'X867', 'X869', 'X870', 'X871', 'X872', 'X104', 'X110', 'X116',\n#             'X122', 'X128', 'X134', 'X140', 'X146', 'X152', 'X158', 'X164', 'X170', 'X176',\n#             'X182', 'X351', 'X357', 'X363', 'X369', 'X375', 'X381', 'X387', 'X393', 'X399',\n#             'X405', 'X411', 'X417', 'X423', 'X429'\n#         ]\n        \n#         # Ensure directories exist\n#         Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n#         Path(self.intermediate_dir).mkdir(parents=True, exist_ok=True)\n\n# class MarketMicrostructureDataProcessor:\n#     \"\"\"Data processing utilities for Market Microstructure pipeline\"\"\"\n#     @staticmethod\n#     def reduce_mem_usage(dataframe: pd.DataFrame, dataset_name: str) -> pd.DataFrame:\n#         \"\"\"Reduce memory usage by downcasting numeric types\"\"\"\n#         print(f'Reducing memory usage for: {dataset_name}')\n#         initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n        \n#         for col in dataframe.columns:\n#             col_type = dataframe[col].dtype\n            \n#             if col_type != object:\n#                 c_min = dataframe[col].min()\n#                 c_max = dataframe[col].max()\n                \n#                 if str(col_type)[:3] == 'int':\n#                     if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n#                         dataframe[col] = dataframe[col].astype(np.int8)\n#                     elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n#                         dataframe[col] = dataframe[col].astype(np.int16)\n#                     elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n#                         dataframe[col] = dataframe[col].astype(np.int32)\n#                     elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n#                         dataframe[col] = dataframe[col].astype(np.int64)\n#                 else:\n#                     if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n#                         dataframe[col] = dataframe[col].astype(np.float16)\n#                     elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n#                         dataframe[col] = dataframe[col].astype(np.float32)\n#                     else:\n#                         dataframe[col] = dataframe[col].astype(np.float64)\n        \n#         final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n#         print(f'Memory usage reduced by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%')\n        \n#         return dataframe\n\n# class MarketMicrostructureFeatureEngineer:\n#     \"\"\"Feature engineering for market microstructure analysis\"\"\"\n#     @staticmethod\n#     def create_features(df: pd.DataFrame) -> pd.DataFrame:\n#         \"\"\"Create extensive market microstructure features\"\"\"\n#         df = df.copy()\n        \n#         # Check if required columns exist\n#         required_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n#         available_cols = [col for col in required_cols if col in df.columns]\n        \n#         if len(available_cols) < len(required_cols):\n#             print(f\"Warning: Only {len(available_cols)} of {len(required_cols)} required columns available\")\n#             print(f\"Missing columns: {set(required_cols) - set(available_cols)}\")\n            \n#             # Create dummy columns if missing\n#             for col in required_cols:\n#                 if col not in df.columns:\n#                     df[col] = 0\n        \n#         # Interaction features\n#         df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n#         df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty']\n#         df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n#         df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n#         df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n#         df['buy_sell_interaction'] = df['buy_qty'] * df['sell_qty']\n        \n#         # Spread indicators\n#         df['spread_indicator'] = (df['ask_qty'] - df['bid_qty']) / (df['ask_qty'] + df['bid_qty'] + 1e-8)\n        \n#         # Volume-weighted features\n#         df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n#         df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n#         df['volume_weighted_bid'] = df['bid_qty'] * df['volume']\n#         df['volume_weighted_ask'] = df['ask_qty'] * df['volume']\n        \n#         # Ratios\n#         df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n#         df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-8)\n        \n#         # Order flow imbalance\n#         df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n        \n#         # Pressure indicators\n#         df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-8)\n#         df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n        \n#         # Liquidity features\n#         df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n#         df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)\n#         df['relative_spread'] = (df['ask_qty'] - df['bid_qty']) / (df['volume'] + 1e-8)\n        \n#         # Trade intensity\n#         df['trade_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-8)\n#         df['avg_trade_size'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n#         df['net_trade_flow'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n        \n#         # Market depth\n#         df['depth_ratio'] = df['total_liquidity'] / (df['volume'] + 1e-8)\n#         df['volume_participation'] = (df['buy_qty'] + df['sell_qty']) / (df['total_liquidity'] + 1e-8)\n#         df['market_activity'] = df['volume'] * df['total_liquidity']\n        \n#         # Spread proxy\n#         df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n#         df['realized_volatility_proxy'] = np.abs(df['order_flow_imbalance']) * df['volume']\n        \n#         # Normalized volumes\n#         df['normalized_buy_volume'] = df['buy_qty'] / (df['bid_qty'] + 1e-8)\n#         df['normalized_sell_volume'] = df['sell_qty'] / (df['ask_qty'] + 1e-8)\n        \n#         # Advanced features\n#         df['liquidity_adjusted_imbalance'] = df['order_flow_imbalance'] * df['depth_ratio']\n#         df['pressure_spread_interaction'] = df['buying_pressure'] * df['spread_indicator']\n        \n#         # Additional market microstructure indicators\n#         df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n#         df['mid_price_proxy'] = (df['bid_qty'] + df['ask_qty']) / 2\n#         df['price_pressure'] = df['net_trade_flow'] * df['volume']\n#         df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-8)\n        \n#         # Volatility and risk proxies\n#         df['volume_volatility'] = df['volume'] * df['spread_indicator']\n#         df['liquidity_risk'] = 1 / (df['total_liquidity'] + 1)\n#         df['execution_risk'] = df['spread_indicator'] * df['liquidity_risk']\n        \n#         # Clean up infinities and NaNs\n#         df = df.replace([np.inf, -np.inf], np.nan)\n#         df = df.fillna(0)\n        \n#         print(f\"Created {len(df.columns)} features including engineered features\")\n        \n#         return df\n\n# class MarketMicrostructureMetaLearning:\n#     \"\"\"Meta-learning utilities for Market Microstructure pipeline\"\"\"\n    \n#     @staticmethod\n#     def create_meta_features(oof_xgb: np.ndarray, oof_ridge: np.ndarray, \n#                            train_labels: np.ndarray) -> pd.DataFrame:\n#         \"\"\"Create comprehensive meta-features for meta-learning\"\"\"\n#         meta_features = {}\n        \n#         # Basic predictions\n#         meta_features['pred_xgboost'] = oof_xgb\n#         meta_features['pred_ridge_ensemble'] = oof_ridge\n        \n#         # Prediction statistics\n#         meta_features['pred_mean'] = (oof_xgb + oof_ridge) / 2\n#         meta_features['pred_diff'] = oof_xgb - oof_ridge\n#         meta_features['pred_abs_diff'] = np.abs(oof_xgb - oof_ridge)\n#         meta_features['pred_max'] = np.maximum(oof_xgb, oof_ridge)\n#         meta_features['pred_min'] = np.minimum(oof_xgb, oof_ridge)\n#         meta_features['pred_range'] = meta_features['pred_max'] - meta_features['pred_min']\n        \n#         # Model agreement\n#         meta_features['model_agreement'] = 1 / (meta_features['pred_abs_diff'] + 1e-8)\n        \n#         # Error features (for training data)\n#         meta_features['error_xgboost'] = oof_xgb - train_labels\n#         meta_features['error_ridge'] = oof_ridge - train_labels\n#         meta_features['abs_error_xgboost'] = np.abs(meta_features['error_xgboost'])\n#         meta_features['abs_error_ridge'] = np.abs(meta_features['error_ridge'])\n#         meta_features['squared_error_xgboost'] = meta_features['error_xgboost'] ** 2\n#         meta_features['squared_error_ridge'] = meta_features['error_ridge'] ** 2\n        \n#         # Best model indicator\n#         meta_features['best_model'] = (meta_features['abs_error_xgboost'] < meta_features['abs_error_ridge']).astype(int)\n#         meta_features['best_model_error'] = np.minimum(meta_features['abs_error_xgboost'], meta_features['abs_error_ridge'])\n#         meta_features['worst_model_error'] = np.maximum(meta_features['abs_error_xgboost'], meta_features['abs_error_ridge'])\n#         meta_features['error_range'] = meta_features['worst_model_error'] - meta_features['best_model_error']\n        \n#         # Rank features\n#         meta_features['rank_xgboost'] = np.argsort(np.argsort(oof_xgb)) / len(oof_xgb)\n#         meta_features['rank_ridge'] = np.argsort(np.argsort(oof_ridge)) / len(oof_ridge)\n        \n#         return pd.DataFrame(meta_features)\n    \n#     @staticmethod\n#     def calculate_performance_segments(oof_xgb: np.ndarray, oof_ridge: np.ndarray,\n#                                      train_labels: np.ndarray) -> Dict[str, Any]:\n#         \"\"\"Calculate model performance across different data segments\"\"\"\n#         performance_segments = {}\n        \n#         # Overall performance\n#         performance_segments['overall'] = {\n#             'xgboost': {\n#                 'correlation': float(pearsonr(oof_xgb, train_labels)[0]),\n#                 'mae': float(np.mean(np.abs(oof_xgb - train_labels))),\n#                 'rmse': float(np.sqrt(np.mean((oof_xgb - train_labels) ** 2)))\n#             },\n#             'ridge_ensemble': {\n#                 'correlation': float(pearsonr(oof_ridge, train_labels)[0]),\n#                 'mae': float(np.mean(np.abs(oof_ridge - train_labels))),\n#                 'rmse': float(np.sqrt(np.mean((oof_ridge - train_labels) ** 2)))\n#             }\n#         }\n        \n#         # Performance by prediction quantiles\n#         pred_mean = (oof_xgb + oof_ridge) / 2\n#         quantiles = np.percentile(pred_mean, [25, 50, 75])\n        \n#         performance_segments['by_prediction_level'] = {}\n#         for q_idx, (low, high) in enumerate([(float('-inf'), quantiles[0]), \n#                                             (quantiles[0], quantiles[1]),\n#                                             (quantiles[1], quantiles[2]),\n#                                             (quantiles[2], float('inf'))]):\n#             mask = (pred_mean >= low) & (pred_mean < high)\n#             if np.sum(mask) > 10:\n#                 performance_segments['by_prediction_level'][f'q{q_idx+1}'] = {\n#                     'xgboost': {\n#                         'correlation': float(pearsonr(oof_xgb[mask], train_labels[mask])[0]) if np.sum(mask) > 1 else 0,\n#                         'mae': float(np.mean(np.abs(oof_xgb[mask] - train_labels[mask]))),\n#                         'n_samples': int(np.sum(mask))\n#                     },\n#                     'ridge_ensemble': {\n#                         'correlation': float(pearsonr(oof_ridge[mask], train_labels[mask])[0]) if np.sum(mask) > 1 else 0,\n#                         'mae': float(np.mean(np.abs(oof_ridge[mask] - train_labels[mask]))),\n#                         'n_samples': int(np.sum(mask))\n#                     }\n#                 }\n        \n#         # Performance by time (first half vs second half)\n#         mid_point = len(train_labels) // 2\n#         performance_segments['by_time'] = {\n#             'first_half': {\n#                 'xgboost': {\n#                     'correlation': float(pearsonr(oof_xgb[:mid_point], train_labels[:mid_point])[0]),\n#                     'mae': float(np.mean(np.abs(oof_xgb[:mid_point] - train_labels[:mid_point])))\n#                 },\n#                 'ridge_ensemble': {\n#                     'correlation': float(pearsonr(oof_ridge[:mid_point], train_labels[:mid_point])[0]),\n#                     'mae': float(np.mean(np.abs(oof_ridge[:mid_point] - train_labels[:mid_point])))\n#                 }\n#             },\n#             'second_half': {\n#                 'xgboost': {\n#                     'correlation': float(pearsonr(oof_xgb[mid_point:], train_labels[mid_point:])[0]),\n#                     'mae': float(np.mean(np.abs(oof_xgb[mid_point:] - train_labels[mid_point:])))\n#                 },\n#                 'ridge_ensemble': {\n#                     'correlation': float(pearsonr(oof_ridge[mid_point:], train_labels[mid_point:])[0]),\n#                     'mae': float(np.mean(np.abs(oof_ridge[mid_point:] - train_labels[mid_point:])))\n#                 }\n#             }\n#         }\n        \n#         # Performance by model disagreement\n#         model_disagreement = np.abs(oof_xgb - oof_ridge)\n#         disagreement_median = np.median(model_disagreement)\n        \n#         performance_segments['by_model_agreement'] = {\n#             'high_agreement': {},\n#             'low_agreement': {}\n#         }\n        \n#         high_agreement_mask = model_disagreement <= disagreement_median\n#         low_agreement_mask = model_disagreement > disagreement_median\n        \n#         for name, mask in [('high_agreement', high_agreement_mask), ('low_agreement', low_agreement_mask)]:\n#             if np.sum(mask) > 10:\n#                 performance_segments['by_model_agreement'][name] = {\n#                     'xgboost': {\n#                         'correlation': float(pearsonr(oof_xgb[mask], train_labels[mask])[0]),\n#                         'mae': float(np.mean(np.abs(oof_xgb[mask] - train_labels[mask]))),\n#                         'n_samples': int(np.sum(mask))\n#                     },\n#                     'ridge_ensemble': {\n#                         'correlation': float(pearsonr(oof_ridge[mask], train_labels[mask])[0]),\n#                         'mae': float(np.mean(np.abs(oof_ridge[mask] - train_labels[mask]))),\n#                         'n_samples': int(np.sum(mask))\n#                     }\n#                 }\n        \n#         return performance_segments\n\n# class MarketMicrostructurePipeline:\n#     \"\"\"Complete Market Microstructure XGBoost Pipeline with Meta-Learning\"\"\"\n#     def __init__(self, config: Optional[MarketMicrostructureConfiguration] = None):\n#         self.config = config or MarketMicrostructureConfiguration()\n#         self.data_processor = MarketMicrostructureDataProcessor()\n#         self.feature_engineer = MarketMicrostructureFeatureEngineer()\n#         self.meta_learning = MarketMicrostructureMetaLearning()\n        \n#     def optimize_ridge_hyperparameters(self, X_ensemble: pd.DataFrame, y: pd.Series) -> Dict[str, Any]:\n#         \"\"\"Optimize Ridge regression hyperparameters using Optuna\"\"\"\n#         print(\"Optimizing Ridge hyperparameters with Optuna...\")\n        \n#         def objective(trial):\n#             params = {\n#                 \"random_state\": self.config.seed,\n#                 \"alpha\": trial.suggest_float(\"alpha\", 0.001, 100),\n#                 \"tol\": trial.suggest_float(\"tol\", 1e-6, 1e-2),\n#                 \"solver\": trial.suggest_categorical(\"solver\", [\"auto\", \"svd\", \"cholesky\", \"lsqr\"])\n#             }\n            \n#             scores = []\n#             kf = KFold(n_splits=self.config.n_folds, shuffle=True, random_state=self.config.seed)\n            \n#             for train_idx, val_idx in kf.split(X_ensemble):\n#                 X_train, X_val = X_ensemble.iloc[train_idx], X_ensemble.iloc[val_idx]\n#                 y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n                \n#                 model = Ridge(**params)\n#                 model.fit(X_train, y_train)\n#                 y_pred = model.predict(X_val)\n                \n#                 score = pearsonr(y_val, y_pred)[0]\n#                 scores.append(score)\n            \n#             return np.mean(scores)\n        \n#         # Set Optuna logging level\n#         optuna.logging.set_verbosity(optuna.logging.WARNING)\n        \n#         sampler = optuna.samplers.TPESampler(seed=self.config.seed, multivariate=True)\n#         study = optuna.create_study(direction=\"maximize\", sampler=sampler)\n#         study.optimize(objective, n_trials=self.config.n_optuna_trials, n_jobs=-1, catch=(ValueError,))\n        \n#         best_params = study.best_params\n#         print(f\"Best Ridge parameters: {best_params}\")\n#         print(f\"Best cross-validation score: {study.best_value:.6f}\")\n        \n#         ridge_params = {\n#             \"random_state\": self.config.seed,\n#             \"alpha\": best_params[\"alpha\"],\n#             \"tol\": best_params[\"tol\"],\n#             \"solver\": best_params.get(\"solver\", \"auto\")\n#         }\n        \n#         # Save Optuna results\n#         optuna_results = {\n#             \"best_params\": best_params,\n#             \"best_value\": study.best_value,\n#             \"n_trials\": len(study.trials),\n#             \"optimization_history\": [trial.value for trial in study.trials if trial.value is not None]\n#         }\n        \n#         with open(self.config.optuna_results_file, 'w') as f:\n#             json.dump(optuna_results, f, indent=2)\n        \n#         # Register Optuna results with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'optuna_results',\n#             self.config.optuna_results_file,\n#             'analysis',\n#             metadata=optuna_results\n#         )\n        \n#         return ridge_params\n    \n#     def save_oof_predictions_and_meta_features(self, cv_results: Dict[str, Any], train_indices: np.ndarray):\n#         \"\"\"Save comprehensive OOF predictions and meta-features for meta-learning\"\"\"\n#         # Create OOF predictions DataFrame\n#         oof_df = pd.DataFrame({\n#             'row_id': train_indices,\n#             'true_label': cv_results['train_labels'],\n#             'fold': cv_results['fold_assignments']\n#         })\n        \n#         # Add predictions from each model\n#         oof_df['pred_xgboost'] = cv_results['oof_xgboost']\n#         oof_df['pred_ridge_ensemble'] = cv_results['oof_ridge']\n        \n#         # Save OOF predictions\n#         oof_df.to_csv(self.config.oof_predictions_path, index=False)\n#         print(f\"OOF predictions saved to {self.config.oof_predictions_path}\")\n        \n#         # Register OOF predictions with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'oof_predictions',\n#             self.config.oof_predictions_path,\n#             'oof_predictions',\n#             metadata={\n#                 'n_models': 2,\n#                 'n_samples': len(oof_df),\n#                 'models': ['xgboost', 'ridge_ensemble']\n#             }\n#         )\n        \n#         # Create and save meta-features\n#         meta_features_df = self.meta_learning.create_meta_features(\n#             cv_results['oof_xgboost'],\n#             cv_results['oof_ridge'],\n#             cv_results['train_labels']\n#         )\n#         meta_features_df['row_id'] = train_indices\n        \n#         # Reorder columns to have row_id first\n#         cols = ['row_id'] + [col for col in meta_features_df.columns if col != 'row_id']\n#         meta_features_df = meta_features_df[cols]\n        \n#         meta_features_df.to_csv(self.config.meta_features_path, index=False)\n#         print(f\"Meta-features saved to {self.config.meta_features_path}\")\n        \n#         # Register meta-features with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'meta_features',\n#             self.config.meta_features_path,\n#             'features',\n#             metadata={\n#                 'n_features': len(meta_features_df.columns) - 1,  # Exclude row_id\n#                 'feature_names': [col for col in meta_features_df.columns if col != 'row_id']\n#             }\n#         )\n        \n#         # Calculate and save row-level metrics\n#         row_metrics = pd.DataFrame({\n#             'row_id': train_indices,\n#             'true_label': cv_results['train_labels']\n#         })\n        \n#         # Add error metrics for each model\n#         row_metrics['error_xgboost'] = cv_results['oof_xgboost'] - cv_results['train_labels']\n#         row_metrics['error_ridge'] = cv_results['oof_ridge'] - cv_results['train_labels']\n#         row_metrics['abs_error_xgboost'] = np.abs(row_metrics['error_xgboost'])\n#         row_metrics['abs_error_ridge'] = np.abs(row_metrics['error_ridge'])\n#         row_metrics['squared_error_xgboost'] = row_metrics['error_xgboost'] ** 2\n#         row_metrics['squared_error_ridge'] = row_metrics['error_ridge'] ** 2\n        \n#         # Add best model indicator\n#         row_metrics['best_model'] = (row_metrics['abs_error_xgboost'] < row_metrics['abs_error_ridge']).astype(int)\n#         row_metrics['best_model_name'] = row_metrics['best_model'].map({0: 'ridge_ensemble', 1: 'xgboost'})\n        \n#         row_metrics.to_csv(self.config.row_level_metrics_path, index=False)\n#         print(f\"Row-level metrics saved to {self.config.row_level_metrics_path}\")\n        \n#         # Register row-level metrics\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'row_level_metrics',\n#             self.config.row_level_metrics_path,\n#             'analysis',\n#             metadata={\n#                 'metrics': [col for col in row_metrics.columns if col not in ['row_id', 'true_label']]\n#             }\n#         )\n    \n#     def save_model_performance_analysis(self, cv_results: Dict[str, Any]):\n#         \"\"\"Save comprehensive model performance analysis\"\"\"\n#         # Calculate performance across different segments\n#         performance_segments = self.meta_learning.calculate_performance_segments(\n#             cv_results['oof_xgboost'],\n#             cv_results['oof_ridge'],\n#             cv_results['train_labels']\n#         )\n        \n#         # Add model metadata\n#         performance_data = {\n#             'model_configurations': {\n#                 'xgboost_params': self.config.xgb_params,\n#                 'ridge_params': cv_results.get('ridge_params', {}),\n#                 'n_features': cv_results.get('n_features', 0),\n#                 'n_engineered_features': cv_results.get('n_engineered_features', 0)\n#             },\n#             'performance_segments': performance_segments,\n#             'cv_settings': {\n#                 'n_folds': self.config.n_folds,\n#                 'shuffle': True,\n#                 'random_state': self.config.seed\n#             },\n#             'feature_engineering': {\n#                 'market_microstructure_features': True,\n#                 'feature_count': cv_results.get('n_features', 0)\n#             }\n#         }\n        \n#         # Save to JSON with custom encoder\n#         with open(self.config.model_performance_path, 'w') as f:\n#             json.dump(performance_data, f, indent=2, cls=NumpyEncoder)\n        \n#         print(f\"Model performance analysis saved to {self.config.model_performance_path}\")\n        \n#         # Register performance analysis\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'model_performance',\n#             self.config.model_performance_path,\n#             'analysis',\n#             metadata={\n#                 'segments_analyzed': list(performance_segments.keys())\n#             }\n#         )\n    \n#     def save_metrics(self, scores: Dict[str, float], ridge_params: Dict[str, Any]):\n#         \"\"\"Save model metrics and parameters\"\"\"\n#         metrics = {\n#             \"model\": \"Market Microstructure XGBoost\",\n#             \"xgboost_score\": scores.get(\"XGBoost\", 0),\n#             \"ridge_ensemble_score\": scores.get(\"ridge_ensemble\", 0),\n#             \"ridge_params\": ridge_params,\n#             \"n_features\": scores.get(\"n_features\", 0),\n#             \"n_folds\": self.config.n_folds\n#         }\n        \n#         metrics_df = pd.DataFrame([metrics])\n#         metrics_df.to_csv(self.config.metrics_file, index=False)\n#         print(f\"Metrics saved to {self.config.metrics_file}\")\n        \n#         # Register metrics with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'model_metrics',\n#             self.config.metrics_file,\n#             'analysis',\n#             metadata=metrics\n#         )\n    \n#     def run(self) -> Tuple[np.ndarray, Dict[str, float]]:\n#         print(\"\\nStarting Market Microstructure XGBoost Pipeline (Enhanced for Meta-Learning)\")\n#         print(\"=\"*80)\n        \n#         # Update model status\n#         global_config.update_model_status(self.config.model_name, \"running\")\n        \n#         # Load data\n#         print(\"\\nLoading data...\")\n#         train = pd.read_parquet(self.config.train_path).reset_index(drop=True)\n#         test = pd.read_parquet(self.config.test_path).reset_index(drop=True)\n        \n#         # Add row indices for tracking\n#         train['original_index'] = range(len(train))\n#         test['original_index'] = range(len(test))\n        \n#         print(f\"Original data shapes - Train: {train.shape}, Test: {test.shape}\")\n        \n#         # Drop unnecessary columns\n#         cols_to_drop_train = [col for col in self.config.cols_to_drop if col in train.columns]\n#         cols_to_drop_test = [col for col in self.config.cols_to_drop + [\"label\"] if col in test.columns]\n        \n#         if cols_to_drop_train:\n#             train = train.drop(columns=cols_to_drop_train)\n#             print(f\"Dropped {len(cols_to_drop_train)} columns from training data\")\n        \n#         if cols_to_drop_test:\n#             test = test.drop(columns=cols_to_drop_test)\n#             print(f\"Dropped {len(cols_to_drop_test)} columns from test data\")\n        \n#         # Reduce memory usage\n#         train = self.data_processor.reduce_mem_usage(train, \"train\")\n#         test = self.data_processor.reduce_mem_usage(test, \"test\")\n        \n#         # Create market microstructure features\n#         print(\"\\nEngineering market microstructure features...\")\n#         train = self.feature_engineer.create_features(train)\n#         test = self.feature_engineer.create_features(test)\n        \n#         # Prepare data for modeling\n#         train_indices = train['original_index'].values\n#         test_indices = test['original_index'].values if 'original_index' in test.columns else range(len(test))\n        \n#         X = train.drop([self.config.target, 'original_index'], axis=1)\n#         y = train[self.config.target]\n#         X_test = test.drop(['original_index'], axis=1) if 'original_index' in test.columns else test\n        \n#         # Clean infinite values\n#         X = X.replace([np.inf, -np.inf], np.nan).fillna(0)\n#         X_test = X_test.replace([np.inf, -np.inf], np.nan).fillna(0)\n        \n#         gc.collect()\n        \n#         print(f\"\\nFinal data shapes - X: {X.shape}, X_test: {X_test.shape}\")\n        \n#         # Initialize storage\n#         scores = {\"n_features\": X.shape[1], \"n_engineered_features\": len([col for col in X.columns if col not in train.columns])}\n#         oof_preds = {}\n#         test_preds = {}\n#         feature_importance_list = []\n#         fold_assignments = np.full(len(X), -1)\n        \n#         print(\"\\nTraining XGBoost with Market Microstructure Features\")\n#         print(f\"Using {self.config.n_folds}-fold cross-validation\")\n        \n#         # Train XGBoost with cross-validation\n#         oof = np.zeros(len(X))\n#         test_predictions = np.zeros(len(X_test))\n#         fold_scores = []\n        \n#         kf = KFold(n_splits=self.config.n_folds, shuffle=True, random_state=self.config.seed)\n        \n#         for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#             print(f\"\\nFold {fold + 1}/{self.config.n_folds}\")\n            \n#             # Store fold assignments\n#             fold_assignments[val_idx] = fold\n            \n#             X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n#             y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n            \n#             print(f\"  Train samples: {len(X_train):,}, Val samples: {len(X_val):,}\")\n            \n#             # Train model\n#             model = XGBRegressor(**self.config.xgb_params)\n#             model.fit(\n#                 X_train, y_train,\n#                 eval_set=[(X_val, y_val)],\n#                 early_stopping_rounds=50,\n#                 verbose=False\n#             )\n            \n#             # Make predictions\n#             y_pred = model.predict(X_val)\n#             oof[val_idx] = y_pred\n            \n#             # Calculate fold score\n#             score = pearsonr(y_val, y_pred)[0]\n#             fold_scores.append(score)\n#             print(f\"  Fold {fold + 1} Score: {score:.6f}\")\n            \n#             # Test predictions\n#             test_predictions += model.predict(X_test) / self.config.n_folds\n            \n#             # Store feature importance\n#             importance = model.feature_importances_\n#             feature_importance_list.append(importance)\n            \n#             # Clean up memory\n#             del X_train, X_val, y_train, y_val, model\n#             gc.collect()\n        \n#         # Calculate overall score\n#         overall_score = pearsonr(y, oof)[0]\n        \n#         print(f\"\\nCross-validation completed:\")\n#         print(f\"  Average fold score: {np.mean(fold_scores):.6f}\")\n#         print(f\"  Overall OOF score: {overall_score:.6f}\")\n        \n#         # Store results\n#         oof_preds[\"XGBoost\"] = oof\n#         test_preds[\"XGBoost\"] = test_predictions\n#         scores[\"XGBoost\"] = overall_score\n        \n#         # Ridge ensemble stacking\n#         print(\"\\nCreating Ridge Regression Ensemble\")\n        \n#         # Prepare ensemble data\n#         X_ensemble = pd.DataFrame(oof_preds)\n#         X_test_ensemble = pd.DataFrame(test_preds)\n        \n#         # Optimize Ridge hyperparameters if enabled\n#         if self.config.run_optuna:\n#             ridge_params = self.optimize_ridge_hyperparameters(X_ensemble, y)\n#         else:\n#             ridge_params = {\"random_state\": self.config.seed, \"alpha\": 1.0}\n#             print(\"Using default Ridge parameters (Optuna disabled)\")\n        \n#         # Train Ridge ensemble with cross-validation\n#         print(\"\\nTraining Ridge ensemble...\")\n#         ridge_test_preds = np.zeros(len(X_test_ensemble))\n#         ridge_oof = np.zeros(len(X_ensemble))\n#         ridge_oof_scores = []\n        \n#         for fold, (train_idx, val_idx) in enumerate(kf.split(X_ensemble)):\n#             X_train, X_val = X_ensemble.iloc[train_idx], X_ensemble.iloc[val_idx]\n#             y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n            \n#             model = Ridge(**ridge_params)\n#             model.fit(X_train, y_train)\n            \n#             # OOF predictions\n#             ridge_oof[val_idx] = model.predict(X_val)\n            \n#             # Validation score\n#             fold_score = pearsonr(y_val, ridge_oof[val_idx])[0]\n#             ridge_oof_scores.append(fold_score)\n            \n#             ridge_test_preds += model.predict(X_test_ensemble) / self.config.n_folds\n        \n#         ridge_ensemble_score = pearsonr(y, ridge_oof)[0]\n#         print(f\"Ridge ensemble OOF score: {ridge_ensemble_score:.6f}\")\n#         scores[\"ridge_ensemble\"] = ridge_ensemble_score\n        \n#         # Prepare CV results for meta-learning\n#         cv_results = {\n#             'oof_xgboost': oof,\n#             'oof_ridge': ridge_oof,\n#             'test_xgboost': test_predictions,\n#             'test_ridge': ridge_test_preds,\n#             'train_labels': y.values,\n#             'fold_assignments': fold_assignments,\n#             'ridge_params': ridge_params,\n#             'n_features': X.shape[1],\n#             'n_engineered_features': len([col for col in X.columns if col not in train.columns])\n#         }\n        \n#         # Save OOF predictions and meta-features\n#         print(\"\\nSaving OOF predictions and meta-features...\")\n#         self.save_oof_predictions_and_meta_features(cv_results, train_indices)\n        \n#         # Save model performance analysis\n#         print(\"\\nSaving model performance analysis...\")\n#         self.save_model_performance_analysis(cv_results)\n        \n#         # Save results\n#         print(\"\\nSaving results...\")\n        \n#         # Save submission\n#         sub = pd.read_csv(self.config.sample_sub_path)\n#         sub[\"prediction\"] = ridge_test_preds\n#         sub.to_csv(self.config.submission_file, index=False)\n#         print(f\"Submission saved to {self.config.submission_file}\")\n        \n#         # Register submission with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'submission',\n#             self.config.submission_file,\n#             'submission',\n#             metadata={\n#                 'xgboost_score': float(overall_score),\n#                 'ridge_ensemble_score': float(ridge_ensemble_score),\n#                 'n_features': X.shape[1],\n#                 'method': 'ridge_stacked_xgboost',\n#                 'has_meta_features': True\n#             }\n#         )\n        \n#         # Save metrics\n#         self.save_metrics(scores, ridge_params)\n        \n#         # Save feature importance\n#         avg_importance = np.mean(feature_importance_list, axis=0)\n#         feature_importance = pd.DataFrame({\n#             'feature': X.columns,\n#             'importance': avg_importance\n#         }).sort_values('importance', ascending=False)\n        \n#         feature_importance.to_csv(self.config.feature_importance_file, index=False)\n        \n#         # Register feature importance with global configuration\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'feature_importance',\n#             self.config.feature_importance_file,\n#             'analysis',\n#             metadata={\n#                 'top_10_features': feature_importance.head(10)['feature'].tolist(),\n#                 'top_10_importances': feature_importance.head(10)['importance'].tolist()\n#             }\n#         )\n        \n#         print(\"\\nMarket Microstructure pipeline completed successfully!\")\n#         print(\"Meta-learning data saved:\")\n#         print(f\"  - OOF predictions: {self.config.oof_predictions_path}\")\n#         print(f\"  - Meta-features: {self.config.meta_features_path}\")\n#         print(f\"  - Row-level metrics: {self.config.row_level_metrics_path}\")\n#         print(f\"  - Model performance: {self.config.model_performance_path}\")\n        \n#         return ridge_test_preds, scores\n\n# # ===========================================================================================\n# # EXECUTION BLOCK\n# # ===========================================================================================\n\n# if __name__ == \"__main__\":\n#     try:\n#         print(\"\\n🏛️ Running Market Microstructure XGBoost Pipeline (Enhanced)\")\n#         print(\"-\"*80)\n        \n#         # Clean memory before starting\n#         aggressive_memory_cleanup()\n        \n#         # Create and run pipeline\n#         market_pipeline = MarketMicrostructurePipeline()\n#         predictions, scores = market_pipeline.run()\n        \n#         # Extract final score (using XGBoost OOF score as primary metric)\n#         final_score = scores.get(\"XGBoost\", 0.0)\n        \n#         # Update global configuration with success\n#         global_config.update_model_status(market_pipeline.config.model_name, 'completed', score=final_score)\n#         print(\"\\n✅ Market Microstructure pipeline completed successfully\")\n        \n#     except Exception as e:\n#         # Update global configuration with failure\n#         error_msg = str(e)\n#         global_config.update_model_status('market_microstructure', 'failed', error_message=error_msg)\n#         print(f\"\\n❌ Market Microstructure pipeline failed: {error_msg}\")\n#         raise\n        \n#     finally:\n#         # Clean up memory\n#         aggressive_memory_cleanup()\n        \n#         # Display current execution status\n#         print(\"\\n\" + \"=\"*80)\n#         print(\"Current Execution Status:\")\n#         print(global_config.get_execution_summary())\n#         print(\"=\"*80)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LIGHTGBM ULTRA MEMORY-OPTIMIZED PIPELINE\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - LightGBM Pipeline\nUltra memory-optimized version with aggressive memory management\n\"\"\"\n\nimport subprocess\nimport sys\nimport os\nimport gc\nimport warnings\nimport json\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr\nfrom typing import List, Dict, Tuple, Optional, Any\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nimport pickle\n\nwarnings.filterwarnings('ignore')\n\n# Check that setup cell was run\ntry:\n    global_config\n    print(\"✓ Global configuration found\")\nexcept NameError:\n    raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# Install required packages\nprint(\"Installing packages for LightGBM pipeline...\")\npackages_to_install = [\n    'lightgbm==4.1.0',\n    'scikit-learn>=1.0.0'\n]\n\nfor package in packages_to_install:\n    try:\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n    except:\n        print(f\"Warning: Could not install {package}\")\n\nimport lightgbm as lgb\nfrom sklearn.base import BaseEstimator, RegressorMixin\n\n# =============================================================================\n# MEMORY MANAGEMENT UTILITIES\n# =============================================================================\n\ndef ultra_aggressive_cleanup():\n    \"\"\"Ultra aggressive memory cleanup\"\"\"\n    for _ in range(5):\n        gc.collect()\n    \ndef get_memory_usage():\n    \"\"\"Get current memory usage in MB\"\"\"\n    import psutil\n    process = psutil.Process()\n    return process.memory_info().rss / 1024 / 1024\n\ndef log_memory(message):\n    \"\"\"Log memory usage with message\"\"\"\n    print(f\"{message}: {get_memory_usage():.2f} MB\")\n\n# =============================================================================\n# UTILITY FUNCTIONS FOR JSON SERIALIZATION\n# =============================================================================\n\ndef convert_numpy_to_python(obj):\n    \"\"\"Convert numpy types to Python native types for JSON serialization\"\"\"\n    if isinstance(obj, np.integer):\n        return int(obj)\n    elif isinstance(obj, np.floating):\n        return float(obj)\n    elif isinstance(obj, np.ndarray):\n        return obj.tolist()\n    elif isinstance(obj, dict):\n        return {key: convert_numpy_to_python(value) for key, value in obj.items()}\n    elif isinstance(obj, list):\n        return [convert_numpy_to_python(item) for item in obj]\n    else:\n        return obj\n\n# =============================================================================\n# MEMORY-EFFICIENT DATA UTILITIES\n# =============================================================================\n\ndef reduce_memory_usage(df, verbose=True):\n    \"\"\"Aggressively reduce memory usage of dataframe\"\"\"\n    if verbose:\n        start_mem = df.memory_usage().sum() / 1024**2\n        print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            # More aggressive type reduction\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                else:\n                    df[col] = df[col].astype(np.int32)  # Avoid int64\n            else:\n                # Always use float32 for floating point\n                df[col] = df[col].astype(np.float32)\n    \n    if verbose:\n        end_mem = df.memory_usage().sum() / 1024**2\n        print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n        print(f'Decreased by {100 * (start_mem - end_mem) / start_mem:.1f}%')\n    \n    return df\n\n# =============================================================================\n# CONFIGURATION (SIMPLIFIED)\n# =============================================================================\n\n@dataclass\nclass LightGBMConfiguration:\n    \"\"\"Configuration for LightGBM Pipeline\"\"\"\n    \n    # Model identification\n    model_name: str = \"lightgbm_optimized\"\n    model_directory: str = \"\"\n    \n    # Feature configuration - REDUCED feature set to save memory\n    base_features: List[str] = field(default_factory=list)\n    engineered_features: List[str] = field(default_factory=list)\n    all_features: List[str] = field(default_factory=list)\n    \n    # LightGBM parameters\n    lgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n    # Training parameters\n    num_boost_round: int = 100  # Reduced from 150\n    n_folds: int = 3  # Reduced from 5\n    decay_factor: float = 0.95\n    verbose_eval: int = 50\n    \n    # Batch processing\n    test_batch_size: int = 50000  # Process test data in batches\n    \n    # Random seed\n    seed: int = 42\n    \n    def __post_init__(self):\n        \"\"\"Initialize configuration and register model\"\"\"\n        if not self.model_directory:\n            self.model_directory = os.path.join(global_config.base_dir, \"lightgbm_optimized\")\n        \n        # Register model with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # Use ONLY the most important features to save memory\n        if not self.base_features:\n            self.base_features = [\n                'X862', 'X344', 'X345', 'X385', 'X137',  # Top 5 from previous runs\n                'X856', 'X598', 'X852', 'X603', 'X674',  # Next 5\n                'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume'\n            ]\n        \n        # Reduced engineered features\n        self.engineered_features = [\n            'bid_ask_interaction', 'buy_sell_interaction',\n            'order_flow_imbalance', 'buying_pressure', 'selling_pressure',\n            'total_liquidity', 'liquidity_imbalance', 'spread_indicator'\n        ]\n        \n        # LightGBM parameters optimized for memory\n        if not self.lgb_params:\n            self.lgb_params = {\n                \"boosting_type\": \"gbdt\",\n                \"objective\": \"regression\",\n                \"metric\": \"mae\",\n                \"colsample_bytree\": 0.8,  # Increased to reduce overfitting with fewer features\n                \"learning_rate\": 0.03,  # Slightly higher with fewer rounds\n                \"min_child_samples\": 50,  # Increased for stability\n                \"max_depth\": 6,  # Limited depth\n                \"n_jobs\": -1,\n                \"num_leaves\": 31,  # Reduced from 64\n                \"random_state\": self.seed,\n                \"reg_alpha\": 50,\n                \"reg_lambda\": 50,\n                \"subsample\": 0.8,\n                \"verbosity\": 1,\n                \"device\": \"cpu\",  # Force CPU to save GPU memory\n                \"max_bin\": 127  # Reduced from default 255\n            }\n        \n        # Output paths\n        self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n        self.models_path = os.path.join(self.model_directory, \"models.pkl\")\n        self.config_path = os.path.join(self.model_directory, \"config.json\")\n        self.feature_importance_path = os.path.join(self.model_directory, \"feature_importance.csv\")\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n        \n        print(f\"Using {len(self.base_features)} base features and {len(self.engineered_features)} engineered features\")\n    \n    def save(self):\n        \"\"\"Save configuration to JSON file\"\"\"\n        config_dict = {\n            'model_name': self.model_name,\n            'base_features': self.base_features,\n            'engineered_features': self.engineered_features,\n            'lgb_params': self.lgb_params,\n            'num_boost_round': self.num_boost_round,\n            'n_folds': self.n_folds,\n            'seed': self.seed\n        }\n        config_dict = convert_numpy_to_python(config_dict)\n        \n        with open(self.config_path, 'w') as f:\n            json.dump(config_dict, f, indent=2)\n\n# =============================================================================\n# ULTRA MEMORY-EFFICIENT FEATURE ENGINEERING\n# =============================================================================\n\nclass UltraMemoryEfficientFeatureEngineer:\n    \"\"\"Ultra memory-efficient feature engineering\"\"\"\n    \n    @staticmethod\n    def engineer_features_minimal(df: pd.DataFrame, features_to_create: List[str]) -> pd.DataFrame:\n        \"\"\"Create only essential engineered features\"\"\"\n        \n        # Create features one at a time and clean up immediately\n        for feature in features_to_create:\n            if feature == 'bid_ask_interaction':\n                df[feature] = (df['bid_qty'] * df['ask_qty']).astype(np.float32)\n            elif feature == 'buy_sell_interaction':\n                df[feature] = (df['buy_qty'] * df['sell_qty']).astype(np.float32)\n            elif feature == 'order_flow_imbalance':\n                df[feature] = ((df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)).astype(np.float32)\n            elif feature == 'buying_pressure':\n                df[feature] = (df['buy_qty'] / (df['volume'] + 1e-8)).astype(np.float32)\n            elif feature == 'selling_pressure':\n                df[feature] = (df['sell_qty'] / (df['volume'] + 1e-8)).astype(np.float32)\n            elif feature == 'total_liquidity':\n                df[feature] = (df['bid_qty'] + df['ask_qty']).astype(np.float32)\n            elif feature == 'liquidity_imbalance':\n                df[feature] = ((df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)).astype(np.float32)\n            elif feature == 'spread_indicator':\n                df[feature] = ((df['ask_qty'] - df['bid_qty']) / (df['ask_qty'] + df['bid_qty'] + 1e-8)).astype(np.float32)\n            \n            # Clean up immediately after each feature\n            df[feature] = df[feature].replace([np.inf, -np.inf], 0).fillna(0)\n        \n        ultra_aggressive_cleanup()\n        return df\n\n# =============================================================================\n# MAIN PIPELINE - ULTRA MEMORY OPTIMIZED\n# =============================================================================\n\nclass UltraMemoryOptimizedLightGBMPipeline:\n    \"\"\"Ultra memory-optimized LightGBM Pipeline\"\"\"\n    \n    def __init__(self, config: Optional[LightGBMConfiguration] = None):\n        self.config = config or LightGBMConfiguration()\n        self.feature_engineer = UltraMemoryEfficientFeatureEngineer()\n        self.models = []\n        self.valid_scores = []\n        \n    def pearsonr_coeff(self, preds, data):\n        \"\"\"Custom Pearson correlation metric for LightGBM\"\"\"\n        y_true = data.get_label()\n        valid_score = pearsonr(y_true, preds)[0]\n        return 'pearsonr_coeff_score', valid_score, True\n    \n    def create_time_based_folds(self, n_samples: int) -> np.ndarray:\n        \"\"\"Create fold assignments without loading full dataframe\"\"\"\n        fold_assignments = np.zeros(n_samples, dtype=np.int8)\n        fold_size = n_samples // (self.config.n_folds + 1)\n        \n        for i in range(self.config.n_folds + 1):\n            start_idx = i * fold_size\n            end_idx = (i + 1) * fold_size if i < self.config.n_folds else n_samples\n            fold_assignments[start_idx:end_idx] = i + 1\n        \n        return fold_assignments\n    \n    def load_data_info(self) -> Tuple[int, int]:\n        \"\"\"Get data dimensions without loading full data\"\"\"\n        # Read just the shape from parquet metadata\n        train_shape = pd.read_parquet(global_config.train_path, columns=['label']).shape\n        test_shape = pd.read_parquet(global_config.test_path, columns=['volume']).shape\n        \n        print(f\"Train samples: {train_shape[0]:,}\")\n        print(f\"Test samples: {test_shape[0]:,}\")\n        \n        return train_shape[0], test_shape[0]\n    \n    def process_fold(self, fold: int, fold_assignments: np.ndarray) -> Tuple[float, np.ndarray, np.ndarray]:\n        \"\"\"Process a single fold with minimal memory usage\"\"\"\n        print(f\"\\n{'='*60}\")\n        print(f\"Processing Fold {fold}/{self.config.n_folds}\")\n        log_memory(\"Start of fold\")\n        \n        # Use last fold as validation\n        valid_fold = self.config.n_folds + 1\n        valid_mask = fold_assignments == valid_fold\n        train_mask = (fold_assignments != valid_fold) & (fold_assignments != fold)\n        \n        # Load only required columns for this fold\n        columns_to_load = ['label'] + self.config.base_features\n        \n        # Load training data for this fold\n        print(\"Loading fold training data...\")\n        train_df = pd.read_parquet(global_config.train_path, columns=columns_to_load)\n        train_df = reduce_memory_usage(train_df, verbose=False)\n        \n        # Apply masks\n        X_train = train_df[train_mask][self.config.base_features]\n        y_train = train_df[train_mask]['label'].values.astype(np.float32)\n        X_valid = train_df[valid_mask][self.config.base_features]\n        y_valid = train_df[valid_mask]['label'].values.astype(np.float32)\n        \n        # Delete full dataframe immediately\n        del train_df\n        ultra_aggressive_cleanup()\n        \n        # Engineer features for train\n        print(\"Engineering features for training data...\")\n        X_train = self.feature_engineer.engineer_features_minimal(X_train, self.config.engineered_features)\n        X_valid = self.feature_engineer.engineer_features_minimal(X_valid, self.config.engineered_features)\n        \n        # Update feature list\n        feature_cols = self.config.base_features + self.config.engineered_features\n        \n        # Create LightGBM datasets with immediate memory release\n        train_data = lgb.Dataset(\n            X_train[feature_cols].values, \n            label=y_train,\n            free_raw_data=True  # Important: free memory after construction\n        )\n        \n        valid_data = lgb.Dataset(\n            X_valid[feature_cols].values, \n            label=y_valid,\n            reference=train_data,\n            free_raw_data=True\n        )\n        \n        # Delete DataFrames before training\n        valid_indices = np.where(valid_mask)[0]\n        del X_train\n        ultra_aggressive_cleanup()\n        log_memory(\"After creating LightGBM datasets\")\n        \n        # Train model\n        print(\"Training model...\")\n        model = lgb.train(\n            self.config.lgb_params,\n            train_data,\n            num_boost_round=self.config.num_boost_round,\n            valid_sets=[valid_data],\n            feval=self.pearsonr_coeff,\n            callbacks=[\n                lgb.callback.log_evaluation(period=self.config.verbose_eval)\n            ]\n        )\n        \n        # Make validation predictions\n        valid_pred = model.predict(X_valid[feature_cols].values, num_iteration=model.best_iteration)\n        valid_score = pearsonr(y_valid, valid_pred)[0]\n        print(f\"Validation Score: {valid_score:.6f}\")\n        \n        # Clean up validation data\n        del X_valid, y_valid, train_data, valid_data\n        ultra_aggressive_cleanup()\n        \n        return model, valid_score, valid_pred, valid_indices, feature_cols\n    \n    def predict_test_batched(self, models: List[Any], feature_cols: List[str]) -> np.ndarray:\n        \"\"\"Make test predictions in batches to save memory\"\"\"\n        print(\"\\nMaking test predictions in batches...\")\n        \n        test_predictions = []\n        n_test = pd.read_parquet(global_config.test_path, columns=['volume']).shape[0]\n        n_batches = (n_test + self.config.test_batch_size - 1) // self.config.test_batch_size\n        \n        for batch_idx in range(n_batches):\n            start_idx = batch_idx * self.config.test_batch_size\n            end_idx = min((batch_idx + 1) * self.config.test_batch_size, n_test)\n            \n            print(f\"Processing test batch {batch_idx + 1}/{n_batches} (rows {start_idx:,}-{end_idx:,})\")\n            \n            # Load batch\n            test_batch = pd.read_parquet(\n                global_config.test_path,\n                columns=self.config.base_features\n            ).iloc[start_idx:end_idx]\n            \n            test_batch = reduce_memory_usage(test_batch, verbose=False)\n            \n            # Engineer features\n            test_batch = self.feature_engineer.engineer_features_minimal(\n                test_batch, \n                self.config.engineered_features\n            )\n            \n            # Make predictions for this batch\n            batch_pred = np.zeros(len(test_batch), dtype=np.float32)\n            for model in models:\n                batch_pred += model.predict(\n                    test_batch[feature_cols].values,\n                    num_iteration=model.best_iteration\n                ) / len(models)\n            \n            test_predictions.append(batch_pred)\n            \n            # Clean up batch\n            del test_batch\n            ultra_aggressive_cleanup()\n        \n        return np.concatenate(test_predictions)\n    \n    def run(self) -> float:\n        \"\"\"Execute the ultra memory-optimized pipeline\"\"\"\n        print(\"\\nStarting Ultra Memory-Optimized LightGBM Pipeline\")\n        print(\"=\"*80)\n        \n        # Update status\n        global_config.update_model_status(self.config.model_name, 'running')\n        \n        try:\n            # Get data info without loading\n            n_train, n_test = self.load_data_info()\n            \n            # Create fold assignments\n            fold_assignments = self.create_time_based_folds(n_train)\n            \n            # Storage for results\n            oof_predictions = np.zeros(n_train, dtype=np.float32)\n            models = []\n            scores = []\n            feature_cols = None\n            \n            # Process each fold sequentially with complete cleanup\n            for fold in range(1, self.config.n_folds + 1):\n                model, score, valid_pred, valid_indices, feature_cols = self.process_fold(\n                    fold, fold_assignments\n                )\n                \n                models.append(model)\n                scores.append(score)\n                oof_predictions[valid_indices] = valid_pred\n                \n                # Force cleanup between folds\n                ultra_aggressive_cleanup()\n                log_memory(f\"After fold {fold}\")\n            \n            # Calculate overall score\n            valid_fold_mask = fold_assignments == (self.config.n_folds + 1)\n            train_labels = pd.read_parquet(\n                global_config.train_path, \n                columns=['label']\n            )['label'].values[valid_fold_mask]\n            \n            overall_score = pearsonr(train_labels, oof_predictions[valid_fold_mask])[0]\n            \n            print(f\"\\n{'='*60}\")\n            print(f\"Average fold score: {np.mean(scores):.6f}\")\n            print(f\"Overall OOF score: {overall_score:.6f}\")\n            \n            # Make test predictions in batches\n            test_predictions = self.predict_test_batched(models, feature_cols)\n            \n            # Save outputs\n            print(\"\\nSaving outputs...\")\n            \n            # Save submission\n            sample_submission = pd.read_csv(global_config.sample_sub_path)\n            sample_submission['prediction'] = test_predictions\n            sample_submission.to_csv(self.config.submission_path, index=False)\n            \n            # Save models\n            with open(self.config.models_path, 'wb') as f:\n                pickle.dump(models, f)\n            \n            # Save feature importance\n            importance_dict = {}\n            for model in models:\n                for idx, feat in enumerate(feature_cols):\n                    if feat not in importance_dict:\n                        importance_dict[feat] = 0\n                    importance_dict[feat] += model.feature_importance(importance_type='gain')[idx]\n            \n            # Average importance\n            for feat in importance_dict:\n                importance_dict[feat] /= len(models)\n            \n            feature_importance_df = pd.DataFrame([\n                {'feature': feat, 'importance': imp} \n                for feat, imp in importance_dict.items()\n            ]).sort_values('importance', ascending=False)\n            \n            feature_importance_df.to_csv(self.config.feature_importance_path, index=False)\n            \n            # Save configuration\n            self.config.save()\n            \n            # Register outputs\n            metadata = convert_numpy_to_python({\n                'score': overall_score,\n                'n_models': len(models),\n                'n_features': len(feature_cols)\n            })\n            \n            global_config.register_model_output(\n                self.config.model_name,\n                'submission',\n                self.config.submission_path,\n                'submission',\n                metadata=metadata\n            )\n            \n            # Update status\n            global_config.update_model_status(\n                self.config.model_name, \n                'completed', \n                score=float(overall_score)\n            )\n            \n            print(f\"\\n✅ Ultra Memory-Optimized LightGBM pipeline completed successfully\")\n            print(f\"  Final Score: {overall_score:.6f}\")\n            print(f\"  Memory Usage: {get_memory_usage():.2f} MB\")\n            print(f\"\\nTop 5 features by importance:\")\n            for _, row in feature_importance_df.head(5).iterrows():\n                print(f\"  {row['feature']}: {row['importance']:.2f}\")\n            \n            return float(overall_score)\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n            print(f\"\\n❌ Pipeline failed: {error_msg}\")\n            raise\n            \n        finally:\n            ultra_aggressive_cleanup()\n            print(\"\\n\" + \"=\"*80)\n            print(\"Current Execution Status:\")\n            print(global_config.get_execution_summary())\n            print(\"=\"*80)\n\n# =============================================================================\n# MAIN EXECUTION\n# =============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\\n🚀 Running Ultra Memory-Optimized LightGBM Pipeline\")\n    print(\"-\"*80)\n    \n    # Import psutil for memory monitoring\n    try:\n        import psutil\n    except:\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"psutil\", \"--quiet\"])\n        import psutil\n    \n    # Clean memory before starting\n    ultra_aggressive_cleanup()\n    log_memory(\"Initial memory\")\n    \n    # Create configuration with reduced parameters\n    config = LightGBMConfiguration(\n        num_boost_round=100,  # Reduced\n        n_folds=3,  # Reduced\n        test_batch_size=50000  # Process test data in batches\n    )\n    \n    # Create and run pipeline\n    pipeline = UltraMemoryOptimizedLightGBMPipeline(config)\n    final_score = pipeline.run()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# XGBOOST FOUR-MODEL ENSEMBLE PIPELINE - ULTRA MEMORY-OPTIMIZED V2\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - XGBoost Four-Model Ensemble Pipeline\nUltra memory-optimized version 2 with aggressive memory management\n\"\"\"\n\nimport subprocess\nimport sys\nimport os\nimport gc\nimport warnings\nimport json\nimport pickle\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr\nfrom typing import List, Dict, Tuple, Optional, Any\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler\n\nwarnings.filterwarnings('ignore')\n\n# Check that setup cell was run\ntry:\n    global_config\n    print(\"✓ Global configuration found\")\nexcept NameError:\n    raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# Install required packages\nprint(\"Installing packages for XGBoost Four-Model Pipeline...\")\npackages_to_install = [\n    'xgboost>=2.0.0',\n    'lightgbm>=4.0.0',\n    'psutil'\n]\n\nfor package in packages_to_install:\n    try:\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n    except:\n        print(f\"Warning: Could not install {package}\")\n\nimport xgboost as xgb\nimport lightgbm as lgb\nimport psutil\n\n# =============================================================================\n# MEMORY MANAGEMENT UTILITIES\n# =============================================================================\n\ndef ultra_aggressive_cleanup():\n    \"\"\"Ultra aggressive memory cleanup\"\"\"\n    for _ in range(5):\n        gc.collect()\n    \ndef get_memory_usage():\n    \"\"\"Get current memory usage in MB\"\"\"\n    process = psutil.Process()\n    return process.memory_info().rss / 1024 / 1024\n\ndef log_memory(message):\n    \"\"\"Log memory usage with message\"\"\"\n    print(f\"{message}: {get_memory_usage():.2f} MB\")\n\ndef convert_numpy_to_python(obj):\n    \"\"\"Convert numpy types to Python native types for JSON serialization\"\"\"\n    if isinstance(obj, np.integer):\n        return int(obj)\n    elif isinstance(obj, np.floating):\n        return float(obj)\n    elif isinstance(obj, np.ndarray):\n        return obj.tolist()\n    elif isinstance(obj, dict):\n        return {key: convert_numpy_to_python(value) for key, value in obj.items()}\n    elif isinstance(obj, list):\n        return [convert_numpy_to_python(item) for item in obj]\n    else:\n        return obj\n\ndef reduce_mem_usage(dataframe, dataset_name=\"\", aggressive=True):\n    \"\"\"Reduce memory usage of dataframe with more aggressive settings\"\"\"\n    print(f'Reducing memory usage for: {dataset_name}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n        \n        if col_type != object:\n            c_min = dataframe[col].min()\n            c_max = dataframe[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    dataframe[col] = dataframe[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    dataframe[col] = dataframe[col].astype(np.int16)\n                else:\n                    dataframe[col] = dataframe[col].astype(np.int32)\n            else:\n                # Always use float32 for aggressive memory saving\n                if aggressive:\n                    dataframe[col] = dataframe[col].astype(np.float32)\n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        dataframe[col] = dataframe[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        dataframe[col] = dataframe[col].astype(np.float32)\n                    else:\n                        dataframe[col] = dataframe[col].astype(np.float64)\n    \n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n    print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n    print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n    \n    return dataframe\n\ndef create_time_weights(n_samples, decay_factor=0.95):\n    \"\"\"Create exponentially decaying weights based on sample position\"\"\"\n    positions = np.arange(n_samples, dtype=np.float32)\n    normalized_positions = positions / (n_samples - 1)\n    weights = decay_factor ** (1 - normalized_positions)\n    weights = weights * n_samples / weights.sum()\n    return weights.astype(np.float32)\n\n# =============================================================================\n# CONFIGURATION\n# =============================================================================\n\n@dataclass\nclass XGBoostFourModelConfiguration:\n    \"\"\"Configuration for XGBoost Four-Model Pipeline\"\"\"\n    \n    # Model identification\n    model_name: str = \"xgboost_four_model_v2\"\n    model_directory: str = \"\"\n    \n    # Feature configuration - REDUCED for memory\n    base_features: List[str] = field(default_factory=list)\n    engineered_features: List[str] = field(default_factory=list)\n    \n    # XGBoost parameters - REDUCED for memory\n    xgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n    # LightGBM parameters - REDUCED for memory\n    lgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n    # Training parameters\n    n_folds: int = 5\n    decay_factor: float = 0.95\n    use_scaler: bool = True\n    early_stopping_rounds: int = 25\n    verbose_eval: int = 200\n    \n    # Memory management\n    batch_size: int = 50000\n    max_features: int = 40  # Limit features for memory\n    \n    # Random seed\n    seed: int = 42\n    \n    def __post_init__(self):\n        \"\"\"Initialize configuration and register model\"\"\"\n        if not self.model_directory:\n            self.model_directory = os.path.join(global_config.base_dir, \"xgboost_four_model_v2\")\n        \n        # Register model with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # REDUCED feature set for memory efficiency\n        if not self.base_features:\n            self.base_features = [\n                \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n                \"X415\", \"X345\", \"X137\", \"X855\", \"X174\",\n                \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n            ]\n        \n        # REDUCED engineered features\n        if not self.engineered_features:\n            self.engineered_features = [\n                'bid_ask_interaction', 'buy_sell_interaction',\n                'order_flow_imbalance', 'buying_pressure', 'selling_pressure',\n                'total_liquidity', 'liquidity_imbalance', 'spread_indicator',\n                'trade_intensity', 'depth_ratio',\n                'normalized_buy_volume', 'normalized_sell_volume',\n                'trade_direction_ratio', 'net_buy_volume',\n                'bid_skew', 'ask_skew'\n            ]\n        \n        # XGBoost parameters - REDUCED for memory\n        if not self.xgb_params:\n            self.xgb_params = {\n                \"tree_method\": \"hist\",\n                \"colsample_bytree\": 0.7,\n                \"learning_rate\": 0.03,\n                \"max_depth\": 6,  # Reduced from 20\n                \"n_estimators\": 500,  # Reduced from 1667\n                \"n_jobs\": 2,  # Limit parallel jobs\n                \"random_state\": self.seed,\n                \"reg_alpha\": 40,\n                \"reg_lambda\": 75,\n                \"subsample\": 0.8,\n                \"verbosity\": 0\n            }\n        \n        # LightGBM parameters - REDUCED for memory\n        if not self.lgb_params:\n            self.lgb_params = {\n                \"boosting_type\": \"gbdt\",\n                \"colsample_bytree\": 0.7,\n                \"learning_rate\": 0.03,\n                \"min_child_samples\": 50,\n                \"n_estimators\": 100,  # Reduced from 126\n                \"n_jobs\": 2,  # Limit parallel jobs\n                \"num_leaves\": 31,  # Reduced from 37\n                \"random_state\": self.seed,\n                \"reg_alpha\": 85,\n                \"reg_lambda\": 99,\n                \"subsample\": 0.8,\n                \"verbose\": -1,\n                \"device\": \"cpu\"\n            }\n        \n        # Output paths\n        self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n        self.config_path = os.path.join(self.model_directory, \"config.json\")\n        self.ensemble_results_path = os.path.join(self.model_directory, \"ensemble_results.csv\")\n        self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n        \n        print(f\"Using {len(self.base_features)} base + {len(self.engineered_features)} engineered features\")\n    \n    def save(self):\n        \"\"\"Save configuration to JSON file\"\"\"\n        config_dict = {\n            'model_name': self.model_name,\n            'base_features': self.base_features,\n            'engineered_features': self.engineered_features,\n            'xgb_params': self.xgb_params,\n            'lgb_params': self.lgb_params,\n            'n_folds': self.n_folds,\n            'seed': self.seed\n        }\n        config_dict = convert_numpy_to_python(config_dict)\n        \n        with open(self.config_path, 'w') as f:\n            json.dump(config_dict, f, indent=2)\n\n# =============================================================================\n# MEMORY-EFFICIENT FEATURE ENGINEERING\n# =============================================================================\n\ndef add_features_minimal(df, feature_list):\n    \"\"\"Add only essential engineered features to save memory\"\"\"\n    # Only create features that are in the feature list\n    if 'bid_ask_interaction' in feature_list:\n        df['bid_ask_interaction'] = (df['bid_qty'] * df['ask_qty']).astype(np.float32)\n    if 'buy_sell_interaction' in feature_list:\n        df['buy_sell_interaction'] = (df['buy_qty'] * df['sell_qty']).astype(np.float32)\n    if 'spread_indicator' in feature_list:\n        df['spread_indicator'] = ((df['ask_qty'] - df['bid_qty']) / (df['ask_qty'] + df['bid_qty'] + 1e-8)).astype(np.float32)\n    if 'order_flow_imbalance' in feature_list:\n        df['order_flow_imbalance'] = ((df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)).astype(np.float32)\n    if 'buying_pressure' in feature_list:\n        df['buying_pressure'] = (df['buy_qty'] / (df['volume'] + 1e-8)).astype(np.float32)\n    if 'selling_pressure' in feature_list:\n        df['selling_pressure'] = (df['sell_qty'] / (df['volume'] + 1e-8)).astype(np.float32)\n    if 'total_liquidity' in feature_list:\n        df['total_liquidity'] = (df['bid_qty'] + df['ask_qty']).astype(np.float32)\n    if 'liquidity_imbalance' in feature_list and 'total_liquidity' in df.columns:\n        df['liquidity_imbalance'] = ((df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)).astype(np.float32)\n    if 'trade_intensity' in feature_list:\n        df['trade_intensity'] = ((df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-8)).astype(np.float32)\n    if 'depth_ratio' in feature_list and 'total_liquidity' in df.columns:\n        df['depth_ratio'] = (df['total_liquidity'] / (df['volume'] + 1e-8)).astype(np.float32)\n    if 'normalized_buy_volume' in feature_list:\n        df['normalized_buy_volume'] = (df['buy_qty'] / (df['bid_qty'] + 1e-8)).astype(np.float32)\n    if 'normalized_sell_volume' in feature_list:\n        df['normalized_sell_volume'] = (df['sell_qty'] / (df['ask_qty'] + 1e-8)).astype(np.float32)\n    if 'trade_direction_ratio' in feature_list:\n        df['trade_direction_ratio'] = (df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-8)).astype(np.float32)\n    if 'net_buy_volume' in feature_list:\n        df['net_buy_volume'] = (df['buy_qty'] - df['sell_qty']).astype(np.float32)\n    if 'bid_skew' in feature_list:\n        df['bid_skew'] = (df['bid_qty'] / (df['bid_qty'] + df['ask_qty'] + 1e-8)).astype(np.float32)\n    if 'ask_skew' in feature_list:\n        df['ask_skew'] = (df['ask_qty'] / (df['bid_qty'] + df['ask_qty'] + 1e-8)).astype(np.float32)\n    \n    # Handle infinities and NaNs\n    df.replace([np.inf, -np.inf], 0, inplace=True)\n    df.fillna(0, inplace=True)\n    \n    return df\n\n# =============================================================================\n# MAIN PIPELINE\n# =============================================================================\n\nclass UltraMemoryOptimizedXGBoostFourModelPipeline:\n    \"\"\"Main XGBoost four-model pipeline with aggressive memory optimization\"\"\"\n    \n    def __init__(self, config: Optional[XGBoostFourModelConfiguration] = None):\n        self.config = config or XGBoostFourModelConfiguration()\n        self.scaler = None\n        self.feature_columns = self.config.base_features + self.config.engineered_features\n        \n    def process_single_fold(self, fold_idx, train_idx, valid_idx, n_train):\n        \"\"\"Process a single fold with minimal memory usage\"\"\"\n        print(f\"\\n{'='*50}\")\n        print(f\"Processing Fold {fold_idx + 1}/{self.config.n_folds}\")\n        print('='*50)\n        log_memory(\"Start of fold\")\n        \n        # Load only base features + label\n        columns_to_load = self.config.base_features + ['label']\n        \n        # Load data in chunks if needed\n        print(\"Loading training data for fold...\")\n        train_df = pd.read_parquet(\n            global_config.train_path,\n            columns=columns_to_load\n        )\n        \n        # Reduce memory immediately\n        train_df = reduce_mem_usage(train_df, \"train_fold\", aggressive=True)\n        \n        # Add engineered features\n        train_df = add_features_minimal(train_df, self.config.engineered_features)\n        \n        # Apply scaler\n        if self.scaler is None:\n            self.scaler = StandardScaler()\n            train_df[self.feature_columns] = self.scaler.fit_transform(\n                train_df[self.feature_columns]\n            ).astype(np.float32)\n        else:\n            train_df[self.feature_columns] = self.scaler.transform(\n                train_df[self.feature_columns]\n            ).astype(np.float32)\n        \n        # Extract fold data\n        X_train = train_df.iloc[train_idx][self.feature_columns].values\n        y_train = train_df.iloc[train_idx]['label'].values.astype(np.float32)\n        X_valid = train_df.iloc[valid_idx][self.feature_columns].values\n        y_valid = train_df.iloc[valid_idx]['label'].values.astype(np.float32)\n        \n        # Initialize predictions for this fold\n        fold_oof_preds = {\n            'model_1': np.zeros(len(valid_idx), dtype=np.float32),\n            'model_2': np.zeros(len(valid_idx), dtype=np.float32),\n            'model_3': np.zeros(len(valid_idx), dtype=np.float32),\n            'model_3_lgb': np.zeros(len(valid_idx), dtype=np.float32)\n        }\n        \n        fold_test_preds = {}\n        \n        # Generate sample weights\n        sample_weights_full = create_time_weights(n_train, self.config.decay_factor)\n        \n        # Calculate cutoffs\n        cutoff_75 = int(n_train * 0.25)\n        cutoff_50 = int(n_train * 0.50)\n        \n        # Model 1: Full Data\n        print(\"\\n--- Model 1: Full Data ---\")\n        train_weights_m1 = sample_weights_full[train_idx]\n        \n        model1 = xgb.XGBRegressor(**self.config.xgb_params)\n        model1.fit(\n            X_train, y_train,\n            sample_weight=train_weights_m1,\n            eval_set=[(X_valid, y_valid)],\n            early_stopping_rounds=self.config.early_stopping_rounds,\n            verbose=False\n        )\n        \n        fold_oof_preds['model_1'] = model1.predict(X_valid)\n        \n        # Predict on test in batches\n        fold_test_preds['model_1'] = self.predict_test_batched(model1)\n        \n        # Clean up model 1\n        del model1\n        ultra_aggressive_cleanup()\n        \n        # Model 2: 75% Recent\n        print(\"\\n--- Model 2: 75% Recent Data ---\")\n        train_idx_recent_75 = train_idx[train_idx >= cutoff_75]\n        \n        if len(train_idx_recent_75) > 0:\n            # Get subset indices\n            train_mask_75 = np.isin(train_idx, train_idx_recent_75)\n            X_train_75 = X_train[train_mask_75]\n            y_train_75 = y_train[train_mask_75]\n            \n            # Adjust weights\n            sample_weights_75 = create_time_weights(n_train - cutoff_75, self.config.decay_factor)\n            train_weights_75 = sample_weights_75[train_idx_recent_75 - cutoff_75]\n            \n            model2 = xgb.XGBRegressor(**self.config.xgb_params)\n            model2.fit(\n                X_train_75, y_train_75,\n                sample_weight=train_weights_75,\n                eval_set=[(X_valid, y_valid)],\n                early_stopping_rounds=self.config.early_stopping_rounds,\n                verbose=False\n            )\n            \n            # Handle predictions\n            valid_mask_75 = valid_idx >= cutoff_75\n            fold_oof_preds['model_2'][valid_mask_75] = model2.predict(X_valid[valid_mask_75])\n            fold_oof_preds['model_2'][~valid_mask_75] = fold_oof_preds['model_1'][~valid_mask_75]\n            \n            fold_test_preds['model_2'] = self.predict_test_batched(model2)\n            \n            # Clean up\n            del model2, X_train_75, y_train_75\n            ultra_aggressive_cleanup()\n        \n        # Model 3: 50% Recent (XGBoost)\n        print(\"\\n--- Model 3: 50% Recent Data (XGBoost) ---\")\n        train_idx_recent_50 = train_idx[train_idx >= cutoff_50]\n        \n        if len(train_idx_recent_50) > 0:\n            # Get subset indices\n            train_mask_50 = np.isin(train_idx, train_idx_recent_50)\n            X_train_50 = X_train[train_mask_50]\n            y_train_50 = y_train[train_mask_50]\n            \n            # Adjust weights\n            sample_weights_50 = create_time_weights(n_train - cutoff_50, self.config.decay_factor)\n            train_weights_50 = sample_weights_50[train_idx_recent_50 - cutoff_50]\n            \n            model3 = xgb.XGBRegressor(**self.config.xgb_params)\n            model3.fit(\n                X_train_50, y_train_50,\n                sample_weight=train_weights_50,\n                eval_set=[(X_valid, y_valid)],\n                early_stopping_rounds=self.config.early_stopping_rounds,\n                verbose=False\n            )\n            \n            # Model 3 LightGBM\n            print(\"\\n--- Model 3: 50% Recent Data (LightGBM) ---\")\n            model3_lgb = lgb.LGBMRegressor(**self.config.lgb_params)\n            model3_lgb.fit(\n                X_train_50, y_train_50,\n                sample_weight=train_weights_50,\n                eval_set=[(X_valid, y_valid)],\n                callbacks=[lgb.log_evaluation(0)]\n            )\n            \n            # Handle predictions\n            valid_mask_50 = valid_idx >= cutoff_50\n            fold_oof_preds['model_3'][valid_mask_50] = model3.predict(X_valid[valid_mask_50])\n            fold_oof_preds['model_3'][~valid_mask_50] = fold_oof_preds['model_1'][~valid_mask_50]\n            \n            fold_oof_preds['model_3_lgb'][valid_mask_50] = model3_lgb.predict(X_valid[valid_mask_50])\n            fold_oof_preds['model_3_lgb'][~valid_mask_50] = fold_oof_preds['model_1'][~valid_mask_50]\n            \n            fold_test_preds['model_3'] = self.predict_test_batched(model3)\n            fold_test_preds['model_3_lgb'] = self.predict_test_batched(model3_lgb)\n            \n            # Clean up\n            del model3, model3_lgb, X_train_50, y_train_50\n            ultra_aggressive_cleanup()\n        \n        # Clean up fold data\n        del train_df, X_train, y_train, X_valid, y_valid\n        ultra_aggressive_cleanup()\n        log_memory(\"End of fold\")\n        \n        return fold_oof_preds, fold_test_preds, valid_idx\n    \n    def predict_test_batched(self, model):\n        \"\"\"Predict on test data in batches to save memory\"\"\"\n        test_predictions = []\n        \n        # Get test size\n        test_info = pd.read_parquet(global_config.test_path, columns=['volume'])\n        n_test = len(test_info)\n        del test_info\n        \n        # Process in batches\n        n_batches = (n_test + self.config.batch_size - 1) // self.config.batch_size\n        \n        for batch_idx in range(n_batches):\n            start_idx = batch_idx * self.config.batch_size\n            end_idx = min((batch_idx + 1) * self.config.batch_size, n_test)\n            \n            # Load batch\n            test_batch = pd.read_parquet(\n                global_config.test_path,\n                columns=self.config.base_features\n            ).iloc[start_idx:end_idx]\n            \n            # Process batch\n            test_batch = reduce_mem_usage(test_batch, \"test_batch\", aggressive=True)\n            test_batch = add_features_minimal(test_batch, self.config.engineered_features)\n            test_batch[self.feature_columns] = self.scaler.transform(\n                test_batch[self.feature_columns]\n            ).astype(np.float32)\n            \n            # Predict\n            batch_pred = model.predict(test_batch[self.feature_columns].values)\n            test_predictions.append(batch_pred)\n            \n            # Clean up\n            del test_batch\n            gc.collect()\n        \n        return np.concatenate(test_predictions)\n    \n    def run(self) -> float:\n        \"\"\"Execute the complete pipeline with minimal memory usage\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"XGBOOST FOUR-MODEL ENSEMBLE PIPELINE (ULTRA MEMORY-OPTIMIZED)\")\n        print(\"=\"*80)\n        \n        # Update status\n        global_config.update_model_status(self.config.model_name, 'running')\n        \n        try:\n            # Get data dimensions without loading\n            train_info = pd.read_parquet(global_config.train_path, columns=['label'])\n            n_train = len(train_info)\n            del train_info\n            \n            test_info = pd.read_parquet(global_config.test_path, columns=['volume'])\n            n_test = len(test_info)\n            del test_info\n            \n            print(f\"Train samples: {n_train:,}\")\n            print(f\"Test samples: {n_test:,}\")\n            \n            # Initialize storage\n            oof_predictions = {\n                'model_1': np.zeros(n_train, dtype=np.float32),\n                'model_2': np.zeros(n_train, dtype=np.float32),\n                'model_3': np.zeros(n_train, dtype=np.float32),\n                'model_3_lgb': np.zeros(n_train, dtype=np.float32)\n            }\n            \n            test_predictions = {\n                'model_1': np.zeros(n_test, dtype=np.float32),\n                'model_2': np.zeros(n_test, dtype=np.float32),\n                'model_3': np.zeros(n_test, dtype=np.float32),\n                'model_3_lgb': np.zeros(n_test, dtype=np.float32)\n            }\n            \n            # Cross-validation\n            kf = KFold(n_splits=self.config.n_folds, shuffle=True, random_state=self.config.seed)\n            \n            # Process each fold sequentially\n            for fold_idx, (train_idx, valid_idx) in enumerate(kf.split(range(n_train))):\n                fold_oof, fold_test, valid_idx = self.process_single_fold(\n                    fold_idx, train_idx, valid_idx, n_train\n                )\n                \n                # Store predictions\n                for model_name in fold_oof:\n                    oof_predictions[model_name][valid_idx] = fold_oof[model_name]\n                    test_predictions[model_name] += fold_test[model_name] / self.config.n_folds\n                \n                # Force cleanup between folds\n                del fold_oof, fold_test\n                ultra_aggressive_cleanup()\n            \n            # Load labels for evaluation\n            train_labels = pd.read_parquet(\n                global_config.train_path, \n                columns=['label']\n            )['label'].values.astype(np.float32)\n            \n            # Calculate scores\n            scores = {}\n            for model_name, preds in oof_predictions.items():\n                scores[model_name] = pearsonr(train_labels, preds)[0]\n            \n            print(\"\\n\" + \"=\" * 50)\n            print(\"INDIVIDUAL MODEL PERFORMANCE\")\n            print(\"=\" * 50)\n            for model_name, score in scores.items():\n                print(f\"{model_name}: {score:.4f}\")\n            \n            # Create ensemble\n            ensemble_oof = np.mean(list(oof_predictions.values()), axis=0)\n            ensemble_test = np.mean(list(test_predictions.values()), axis=0)\n            ensemble_score = pearsonr(train_labels, ensemble_oof)[0]\n            \n            # Weighted ensemble\n            total_score = sum(scores.values())\n            weights = {k: v / total_score for k, v in scores.items()}\n            \n            weighted_oof = sum(weights[k] * v for k, v in oof_predictions.items())\n            weighted_test = sum(weights[k] * v for k, v in test_predictions.items())\n            weighted_score = pearsonr(train_labels, weighted_oof)[0]\n            \n            print(\"\\n\" + \"=\" * 50)\n            print(\"ENSEMBLE PERFORMANCE\")\n            print(\"=\" * 50)\n            print(f\"Simple Ensemble: {ensemble_score:.4f}\")\n            print(f\"Weighted Ensemble: {weighted_score:.4f}\")\n            \n            # Use better ensemble\n            if weighted_score > ensemble_score:\n                final_predictions = weighted_test\n                final_score = weighted_score\n                print(\"\\nUsing weighted ensemble\")\n            else:\n                final_predictions = ensemble_test\n                final_score = ensemble_score\n                print(\"\\nUsing simple ensemble\")\n            \n            # Save submission\n            sample = pd.read_csv(global_config.sample_sub_path)\n            sample['prediction'] = final_predictions\n            sample.to_csv(self.config.submission_path, index=False)\n            print(f\"\\nSubmission saved to {self.config.submission_path}\")\n            \n            # Save results\n            results = pd.DataFrame({\n                'model': list(scores.keys()) + ['simple_ensemble', 'weighted_ensemble'],\n                'score': list(scores.values()) + [ensemble_score, weighted_score]\n            })\n            results.to_csv(self.config.ensemble_results_path, index=False)\n            \n            # Save OOF predictions\n            oof_df = pd.DataFrame({\n                'row_id': range(n_train),\n                'true_label': train_labels,\n                **{f'pred_{k}': v for k, v in oof_predictions.items()}\n            })\n            oof_df.to_csv(self.config.oof_predictions_path, index=False)\n            \n            # Clean up\n            del train_labels, oof_predictions, test_predictions\n            ultra_aggressive_cleanup()\n            \n            # Save configuration\n            self.config.save()\n            \n            # Register outputs\n            global_config.register_model_output(\n                self.config.model_name,\n                'submission',\n                self.config.submission_path,\n                'submission',\n                metadata={'score': float(final_score), 'n_models': 4}\n            )\n            \n            # Update status\n            global_config.update_model_status(\n                self.config.model_name, \n                'completed', \n                score=float(final_score)\n            )\n            \n            print(f\"\\n✅ Pipeline completed successfully\")\n            print(f\"  Final Score: {final_score:.6f}\")\n            print(f\"  Memory Usage: {get_memory_usage():.2f} MB\")\n            \n            return float(final_score)\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n            print(f\"\\n❌ Pipeline failed: {error_msg}\")\n            raise\n            \n        finally:\n            ultra_aggressive_cleanup()\n            print(\"\\n\" + \"=\"*80)\n            print(\"Current Execution Status:\")\n            print(global_config.get_execution_summary())\n            print(\"=\"*80)\n\n# =============================================================================\n# MAIN EXECUTION\n# =============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\\n🚀 Running XGBoost Four-Model Ensemble Pipeline (Ultra Memory-Optimized)\")\n    print(\"-\"*80)\n    \n    # Clean memory before starting\n    ultra_aggressive_cleanup()\n    log_memory(\"Initial memory\")\n    \n    # Create configuration with memory-efficient settings\n    config = XGBoostFourModelConfiguration(\n        n_folds=5,\n        decay_factor=0.95,\n        use_scaler=True,\n        batch_size=50000\n    )\n    \n    # Create and run pipeline\n    pipeline = UltraMemoryOptimizedXGBoostFourModelPipeline(config)\n    final_score = pipeline.run()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # MEMORY-OPTIMIZED HYBRID ENSEMBLE PIPELINE - FULLY ENCAPSULATED IMPLEMENTATION\n# # !/usr/bin/env python\n# # -*- coding: utf-8 -*-\n# \"\"\"\n# DRW Crypto Market Prediction - Memory-Optimized Hybrid Ensemble Pipeline\n# Advanced feature engineering with dimensionality reduction and multi-model ensemble\n# \"\"\"\n\n# import subprocess\n# import sys\n# import os\n# import gc\n# import warnings\n# import json\n# import pickle\n# import pandas as pd\n# import numpy as np\n# from scipy.stats import pearsonr\n# from typing import List, Dict, Tuple, Optional, Any\n# from pathlib import Path\n# from dataclasses import dataclass, field\n# from datetime import datetime\n# from sklearn.preprocessing import StandardScaler, QuantileTransformer\n# from sklearn.model_selection import train_test_split\n# from sklearn.feature_selection import SelectKBest, f_regression\n# from sklearn.decomposition import IncrementalPCA, TruncatedSVD\n# from sklearn.ensemble import RandomForestRegressor\n\n# warnings.filterwarnings('ignore')\n\n# # Check that setup cell was run\n# try:\n#     global_config\n#     print(\"✓ Global configuration found\")\n# except NameError:\n#     raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# # Install required packages\n# print(\"Installing packages for Memory-Optimized Hybrid pipeline...\")\n# packages_to_install = [\n#     'xgboost==2.0.3',\n#     'lightgbm==4.1.0',\n#     'scikit-learn>=1.0.0',\n#     'umap-learn==0.5.4',\n#     'numba>=0.57.0'  # Required for UMAP\n# ]\n\n# for package in packages_to_install:\n#     try:\n#         subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n#     except Exception as e:\n#         print(f\"Warning: Could not install {package}: {e}\")\n\n# # Import packages after installation\n# import xgboost as xgb\n# from xgboost import XGBRegressor\n# import lightgbm as lgb\n\n# try:\n#     import umap\n#     UMAP_AVAILABLE = True\n# except ImportError:\n#     print(\"Warning: UMAP not available, will use fallback\")\n#     UMAP_AVAILABLE = False\n\n# # =============================================================================\n# # CONFIGURATION\n# # =============================================================================\n\n# @dataclass\n# class MemoryOptimizedHybridConfiguration:\n#     \"\"\"Configuration for Memory-Optimized Hybrid Pipeline\"\"\"\n    \n#     # Model identification\n#     model_name: str = \"memory_optimized_hybrid\"\n#     model_directory: str = \"\"\n    \n#     # Memory optimization parameters\n#     chunk_size: int = 50000\n#     feature_selection_sample: int = 150000\n#     training_sample: int = 350000\n#     dtype_reduce: bool = True\n    \n#     # Feature engineering parameters\n#     n_top_features: int = 200\n#     n_interactions: int = 10\n    \n#     # Dimensionality reduction parameters\n#     n_pca_components: int = 40\n#     n_svd_components: int = 30\n#     n_umap_components: int = 50\n#     umap_fit_sample: int = 10000\n#     umap_mini_batch_size: int = 5000\n    \n#     # Model parameters\n#     xgb_params: Dict[str, Any] = field(default_factory=dict)\n#     lgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n#     # Training parameters\n#     validation_size: float = 0.2\n#     random_state: int = 42\n    \n#     def __post_init__(self):\n#         \"\"\"Initialize configuration and register model\"\"\"\n#         if not self.model_directory:\n#             self.model_directory = os.path.join(global_config.base_dir, \"memory_optimized_hybrid\")\n        \n#         # Register model with global configuration\n#         global_config.register_model(self.model_name, self.model_directory)\n        \n#         # Default XGBoost parameters\n#         if not self.xgb_params:\n#             self.xgb_params = {\n#                 'n_estimators': 300,\n#                 'max_depth': 6,\n#                 'learning_rate': 0.05,\n#                 'subsample': 0.8,\n#                 'colsample_bytree': 0.8,\n#                 'tree_method': 'hist',\n#                 'random_state': self.random_state,\n#                 'n_jobs': -1,\n#                 'reg_alpha': 0.1,\n#                 'reg_lambda': 1.0\n#             }\n        \n#         # Default LightGBM parameters\n#         if not self.lgb_params:\n#             self.lgb_params = {\n#                 'objective': 'regression',\n#                 'metric': 'rmse',\n#                 'num_leaves': 31,\n#                 'learning_rate': 0.05,\n#                 'feature_fraction': 0.8,\n#                 'bagging_fraction': 0.8,\n#                 'bagging_freq': 5,\n#                 'random_state': self.random_state,\n#                 'n_jobs': -1,\n#                 'force_col_wise': True\n#             }\n        \n#         # Output paths\n#         self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n#         self.models_path = os.path.join(self.model_directory, \"models.pkl\")\n#         self.config_path = os.path.join(self.model_directory, \"config.json\")\n#         self.feature_list_path = os.path.join(self.model_directory, \"selected_features.json\")\n        \n#         # Meta-learning data paths\n#         self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n#         self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n#         self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n#         self.feature_importance_path = os.path.join(self.model_directory, \"feature_importance.csv\")\n#         self.dimensionality_reduction_path = os.path.join(self.model_directory, \"dim_reduction_models.pkl\")\n        \n#         # Ensure directories exist\n#         Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n    \n#     def save(self):\n#         \"\"\"Save configuration to JSON file\"\"\"\n#         config_dict = {\n#             'model_name': self.model_name,\n#             'chunk_size': self.chunk_size,\n#             'feature_selection_sample': self.feature_selection_sample,\n#             'training_sample': self.training_sample,\n#             'n_top_features': self.n_top_features,\n#             'n_pca_components': self.n_pca_components,\n#             'n_svd_components': self.n_svd_components,\n#             'n_umap_components': self.n_umap_components,\n#             'xgb_params': self.xgb_params,\n#             'lgb_params': self.lgb_params,\n#             'random_state': self.random_state\n#         }\n#         with open(self.config_path, 'w') as f:\n#             json.dump(config_dict, f, indent=2)\n\n# # =============================================================================\n# # DATA PROCESSING\n# # =============================================================================\n\n# class MemoryEfficientDataProcessor:\n#     \"\"\"Memory-efficient data processing utilities\"\"\"\n    \n#     def __init__(self, config: MemoryOptimizedHybridConfiguration):\n#         self.config = config\n    \n#     def clean_financial_data_chunked(self, df: pd.DataFrame, feature_cols: List[str]) -> pd.DataFrame:\n#         \"\"\"Memory-efficient data cleaning with chunked processing\"\"\"\n#         print(f\"Cleaning {len(df):,} rows with {len(feature_cols)} features...\")\n        \n#         cleaned_chunks = []\n        \n#         for chunk_start in range(0, len(df), self.config.chunk_size):\n#             chunk_end = min(chunk_start + self.config.chunk_size, len(df))\n#             chunk = df.iloc[chunk_start:chunk_end][feature_cols].copy()\n            \n#             # Replace infinity values\n#             chunk = chunk.replace([np.inf, -np.inf], np.nan)\n            \n#             # Simple quantile clipping per chunk\n#             for col in chunk.columns:\n#                 col_data = chunk[col].dropna()\n#                 if len(col_data) > 0:\n#                     lower = col_data.quantile(0.001)\n#                     upper = col_data.quantile(0.999)\n#                     chunk[col] = chunk[col].clip(lower=lower, upper=upper)\n            \n#             # Fill missing values with 0\n#             chunk = chunk.fillna(0)\n            \n#             # Convert to float32 to save memory\n#             if self.config.dtype_reduce:\n#                 chunk = chunk.astype(np.float32)\n            \n#             cleaned_chunks.append(chunk)\n            \n#             # Progress update\n#             if chunk_start % (self.config.chunk_size * 5) == 0:\n#                 print(f\"  Cleaned {chunk_start:,} rows...\")\n#                 gc.collect()\n        \n#         # Concatenate all chunks\n#         result = pd.concat(cleaned_chunks, axis=0)\n#         del cleaned_chunks\n#         gc.collect()\n        \n#         return result\n    \n#     def create_lightweight_features(self, X: pd.DataFrame, n_interactions: int = 10) -> pd.DataFrame:\n#         \"\"\"Create a minimal set of high-value engineered features\"\"\"\n#         print(\"Creating lightweight engineered features...\")\n        \n#         X_array = X.values\n#         columns = X.columns\n        \n#         new_features = []\n#         new_names = []\n        \n#         # 1. Simple statistics on feature groups\n#         for i in range(0, X_array.shape[1], 50):\n#             end_idx = min(i + 10, X_array.shape[1])\n#             if end_idx > i:\n#                 # Row-wise mean\n#                 new_features.append(np.mean(X_array[:, i:end_idx], axis=1))\n#                 new_names.append(f'mean_group_{i}')\n                \n#                 # Row-wise std\n#                 new_features.append(np.std(X_array[:, i:end_idx], axis=1))\n#                 new_names.append(f'std_group_{i}')\n        \n#         # 2. Top variance features squared\n#         variances = np.var(X_array, axis=0)\n#         top_var_indices = np.argsort(variances)[-5:]\n        \n#         for idx in top_var_indices:\n#             new_features.append(X_array[:, idx] ** 2)\n#             new_names.append(f'{columns[idx]}_squared')\n        \n#         # 3. Limited interactions based on correlation sampling\n#         sample_size = min(10000, X_array.shape[0])\n#         X_sample = X_array[:sample_size]\n        \n#         # Fast correlation calculation\n#         X_sample_centered = X_sample - X_sample.mean(axis=0)\n#         cov = np.dot(X_sample_centered.T, X_sample_centered) / (sample_size - 1)\n#         std = np.sqrt(np.diag(cov))\n#         corr = cov / np.outer(std, std)\n        \n#         # Find top correlated pairs\n#         np.fill_diagonal(corr, 0)\n#         corr_flat = np.abs(corr).flatten()\n#         top_indices_flat = np.argpartition(corr_flat, -n_interactions)[-n_interactions:]\n#         top_indices = np.unravel_index(top_indices_flat, corr.shape)\n        \n#         added_pairs = set()\n#         for i, j in zip(top_indices[0], top_indices[1]):\n#             if i < j and (i, j) not in added_pairs and len(added_pairs) < n_interactions // 2:\n#                 added_pairs.add((i, j))\n#                 new_features.append(X_array[:, i] * X_array[:, j])\n#                 new_names.append(f'{columns[i]}_x_{columns[j]}')\n        \n#         # Combine features\n#         if new_features:\n#             new_features_array = np.column_stack(new_features).astype(np.float32)\n#             new_df = pd.DataFrame(new_features_array, columns=new_names, index=X.index)\n#             result = pd.concat([X, new_df], axis=1)\n#             print(f\"  Created {len(new_features)} new features\")\n#             return result\n        \n#         return X\n\n# # =============================================================================\n# # FEATURE SELECTION\n# # =============================================================================\n\n# class MemoryEfficientFeatureSelector:\n#     \"\"\"Memory-efficient feature selection\"\"\"\n    \n#     def __init__(self, config: MemoryOptimizedHybridConfiguration):\n#         self.config = config\n    \n#     def select_features(self, X: pd.DataFrame, y: np.ndarray, n_features: int = 150) -> Tuple[List[str], np.ndarray]:\n#         \"\"\"Memory-efficient feature selection using sampling\"\"\"\n#         print(f\"\\nSelecting top {n_features} features using sampling strategy...\")\n        \n#         # Use sampling for feature selection\n#         sample_size = min(self.config.feature_selection_sample, len(X))\n#         sample_indices = np.random.choice(len(X), sample_size, replace=False)\n        \n#         X_sample = X.iloc[sample_indices]\n#         y_sample = y[sample_indices]\n        \n#         # 1. F-statistic\n#         print(\"  Computing F-statistics...\")\n#         f_selector = SelectKBest(score_func=f_regression, k=n_features)\n#         f_selector.fit(X_sample, y_sample)\n#         f_scores = f_selector.scores_\n        \n#         # 2. Simplified Random Forest\n#         print(\"  Computing RF importance on subsample...\")\n#         rf_sample_size = min(30000, len(X_sample))\n#         rf_indices = np.random.choice(len(X_sample), rf_sample_size, replace=False)\n        \n#         X_rf = X_sample.iloc[rf_indices]\n#         y_rf = y_sample[rf_indices]\n        \n#         rf = RandomForestRegressor(\n#             n_estimators=50,\n#             max_depth=5,\n#             max_features='sqrt',\n#             random_state=self.config.random_state,\n#             n_jobs=-1\n#         )\n#         rf.fit(X_rf, y_rf)\n#         rf_scores = rf.feature_importances_\n        \n#         # Clear RF model from memory\n#         del rf, X_rf, y_rf\n#         gc.collect()\n        \n#         # Normalize and combine scores\n#         f_scores_norm = (f_scores - f_scores.min()) / (f_scores.max() - f_scores.min() + 1e-8)\n#         rf_scores_norm = (rf_scores - rf_scores.min()) / (rf_scores.max() - rf_scores.min() + 1e-8)\n        \n#         # Weighted combination\n#         combined_scores = 0.6 * f_scores_norm + 0.4 * rf_scores_norm\n        \n#         # Select top features\n#         top_indices = np.argsort(combined_scores)[-n_features:]\n#         selected_features = X.columns[top_indices].tolist()\n        \n#         print(f\"\\n  Top 10 features:\")\n#         feature_scores = [(X.columns[i], combined_scores[i]) for i in top_indices[-10:]]\n#         for i, (feat, score) in enumerate(reversed(feature_scores)):\n#             print(f\"    {i+1}. {feat}: {score:.3f}\")\n        \n#         # Clean up\n#         del X_sample, y_sample\n#         gc.collect()\n        \n#         return selected_features, combined_scores\n\n# # =============================================================================\n# # DIMENSIONALITY REDUCTION\n# # =============================================================================\n\n# class DimensionalityReducer:\n#     \"\"\"Memory-efficient dimensionality reduction\"\"\"\n    \n#     def __init__(self, config: MemoryOptimizedHybridConfiguration):\n#         self.config = config\n#         self.models = {}\n    \n#     def fit_transform(self, X_train: np.ndarray, X_test: np.ndarray, y_train: np.ndarray) -> Dict[str, Dict[str, np.ndarray]]:\n#         \"\"\"Apply multiple dimensionality reduction techniques\"\"\"\n#         print(\"\\nApplying memory-efficient dimensionality reduction...\")\n        \n#         reduced_features = {}\n        \n#         # 1. Incremental PCA\n#         print(\"1. Applying Incremental PCA...\")\n#         try:\n#             X_train_pca, X_test_pca = self._apply_incremental_pca(X_train, X_test)\n#             reduced_features['pca'] = {\n#                 'train': X_train_pca,\n#                 'test': X_test_pca,\n#                 'n_components': self.config.n_pca_components\n#             }\n#         except Exception as e:\n#             print(f\"  PCA failed: {e}\")\n        \n#         # 2. Truncated SVD\n#         print(\"2. Applying Truncated SVD...\")\n#         try:\n#             X_train_svd, X_test_svd = self._apply_truncated_svd(X_train, X_test)\n#             reduced_features['svd'] = {\n#                 'train': X_train_svd,\n#                 'test': X_test_svd,\n#                 'n_components': self.config.n_svd_components\n#             }\n#         except Exception as e:\n#             print(f\"  SVD failed: {e}\")\n        \n#         # 3. UMAP\n#         if UMAP_AVAILABLE:\n#             print(\"3. Applying memory-optimized UMAP...\")\n#             try:\n#                 X_train_umap, X_test_umap = self._apply_umap(X_train, X_test, y_train)\n#                 reduced_features['umap'] = {\n#                     'train': X_train_umap,\n#                     'test': X_test_umap,\n#                     'n_components': self.config.n_umap_components\n#                 }\n#             except Exception as e:\n#                 print(f\"  UMAP failed: {e}\")\n#                 # Fallback to random projection\n#                 X_train_umap, X_test_umap = self._apply_random_projection(X_train, X_test)\n#                 reduced_features['umap'] = {\n#                     'train': X_train_umap,\n#                     'test': X_test_umap,\n#                     'n_components': self.config.n_umap_components\n#                 }\n#         else:\n#             print(\"3. UMAP not available, using random projection...\")\n#             X_train_umap, X_test_umap = self._apply_random_projection(X_train, X_test)\n#             reduced_features['umap'] = {\n#                 'train': X_train_umap,\n#                 'test': X_test_umap,\n#                 'n_components': self.config.n_umap_components\n#             }\n        \n#         return reduced_features\n    \n#     def _apply_incremental_pca(self, X_train: np.ndarray, X_test: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:\n#         \"\"\"Apply Incremental PCA with chunking\"\"\"\n#         # Fit scaler in chunks\n#         scaler = StandardScaler()\n#         for i in range(0, len(X_train), self.config.chunk_size):\n#             chunk = X_train[i:i+self.config.chunk_size]\n#             if i == 0:\n#                 scaler.fit(chunk)\n#             else:\n#                 scaler.partial_fit(chunk)\n        \n#         # Apply Incremental PCA\n#         ipca = IncrementalPCA(n_components=self.config.n_pca_components, batch_size=self.config.chunk_size)\n        \n#         # Fit PCA in chunks\n#         for i in range(0, len(X_train), self.config.chunk_size):\n#             chunk = X_train[i:i+self.config.chunk_size]\n#             chunk_scaled = scaler.transform(chunk)\n#             ipca.partial_fit(chunk_scaled)\n#             del chunk_scaled\n#             gc.collect()\n        \n#         # Transform data in chunks\n#         train_pca_chunks = []\n#         for i in range(0, len(X_train), self.config.chunk_size):\n#             chunk = X_train[i:i+self.config.chunk_size]\n#             chunk_scaled = scaler.transform(chunk)\n#             chunk_pca = ipca.transform(chunk_scaled)\n#             train_pca_chunks.append(chunk_pca)\n#             del chunk_scaled\n#             gc.collect()\n        \n#         X_train_pca = np.vstack(train_pca_chunks)\n#         del train_pca_chunks\n#         gc.collect()\n        \n#         # Transform test data\n#         test_pca_chunks = []\n#         for i in range(0, len(X_test), self.config.chunk_size):\n#             chunk = X_test[i:i+self.config.chunk_size]\n#             chunk_scaled = scaler.transform(chunk)\n#             chunk_pca = ipca.transform(chunk_scaled)\n#             test_pca_chunks.append(chunk_pca)\n#             del chunk_scaled\n#             gc.collect()\n        \n#         X_test_pca = np.vstack(test_pca_chunks)\n#         del test_pca_chunks\n#         gc.collect()\n        \n#         print(f\"  PCA explained variance: {ipca.explained_variance_ratio_.sum():.3f}\")\n        \n#         self.models['pca_scaler'] = scaler\n#         self.models['pca'] = ipca\n        \n#         return X_train_pca, X_test_pca\n    \n#     def _apply_truncated_svd(self, X_train: np.ndarray, X_test: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:\n#         \"\"\"Apply Truncated SVD\"\"\"\n#         svd = TruncatedSVD(n_components=self.config.n_svd_components, random_state=self.config.random_state)\n#         X_train_svd = svd.fit_transform(X_train)\n#         X_test_svd = svd.transform(X_test)\n        \n#         self.models['svd'] = svd\n        \n#         return X_train_svd, X_test_svd\n    \n#     def _apply_umap(self, X_train: np.ndarray, X_test: np.ndarray, y_train: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:\n#         \"\"\"Apply UMAP with extreme memory optimization\"\"\"\n#         # Use very small sample for UMAP fitting\n#         sample_indices = np.random.choice(len(X_train), self.config.umap_fit_sample, replace=False)\n#         X_train_sample = X_train[sample_indices]\n        \n#         # Create scaler for UMAP\n#         umap_scaler = StandardScaler()\n#         X_train_sample_scaled = umap_scaler.fit_transform(X_train_sample)\n#         X_train_sample_scaled = np.ascontiguousarray(X_train_sample_scaled.astype(np.float32))\n        \n#         # Configure UMAP\n#         umap_model = umap.UMAP(\n#             n_components=self.config.n_umap_components,\n#             n_neighbors=15,\n#             min_dist=0.1,\n#             metric='euclidean',\n#             random_state=self.config.random_state,\n#             low_memory=False,\n#             n_epochs=100,\n#             negative_sample_rate=2,\n#             transform_queue_size=2.0,\n#             verbose=False\n#         )\n        \n#         # Fit on sample\n#         print(f\"  Fitting UMAP on {self.config.umap_fit_sample:,} samples...\")\n#         umap_model.fit(X_train_sample_scaled)\n        \n#         # Clear sample\n#         del X_train_sample, X_train_sample_scaled\n#         gc.collect()\n        \n#         # Transform training data in mini-batches\n#         print(\"  Transforming training data...\")\n#         train_umap_chunks = []\n        \n#         for i in range(0, len(X_train), self.config.umap_mini_batch_size):\n#             batch_end = min(i + self.config.umap_mini_batch_size, len(X_train))\n#             batch = X_train[i:batch_end]\n            \n#             batch_scaled = umap_scaler.transform(batch)\n#             batch_scaled = np.ascontiguousarray(batch_scaled.astype(np.float32))\n#             batch_umap = umap_model.transform(batch_scaled)\n#             train_umap_chunks.append(batch_umap.astype(np.float32))\n            \n#             del batch, batch_scaled, batch_umap\n#             gc.collect()\n        \n#         X_train_umap = np.vstack(train_umap_chunks)\n#         del train_umap_chunks\n#         gc.collect()\n        \n#         # Transform test data\n#         print(\"  Transforming test data...\")\n#         test_umap_chunks = []\n        \n#         for i in range(0, len(X_test), self.config.umap_mini_batch_size):\n#             batch_end = min(i + self.config.umap_mini_batch_size, len(X_test))\n#             batch = X_test[i:batch_end]\n            \n#             batch_scaled = umap_scaler.transform(batch)\n#             batch_scaled = np.ascontiguousarray(batch_scaled.astype(np.float32))\n#             batch_umap = umap_model.transform(batch_scaled)\n#             test_umap_chunks.append(batch_umap.astype(np.float32))\n            \n#             del batch, batch_scaled, batch_umap\n#             gc.collect()\n        \n#         X_test_umap = np.vstack(test_umap_chunks)\n#         del test_umap_chunks\n#         gc.collect()\n        \n#         self.models['umap_scaler'] = umap_scaler\n#         self.models['umap'] = umap_model\n        \n#         print(\"  UMAP completed successfully\")\n        \n#         return X_train_umap, X_test_umap\n    \n#     def _apply_random_projection(self, X_train: np.ndarray, X_test: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:\n#         \"\"\"Simple random projection fallback\"\"\"\n#         np.random.seed(self.config.random_state)\n#         projection_matrix = np.random.randn(X_train.shape[1], self.config.n_umap_components).astype(np.float32)\n#         projection_matrix /= np.linalg.norm(projection_matrix, axis=0)\n        \n#         X_train_proj = np.dot(X_train, projection_matrix)\n#         X_test_proj = np.dot(X_test, projection_matrix)\n        \n#         self.models['random_projection'] = projection_matrix\n        \n#         return X_train_proj, X_test_proj\n\n# # =============================================================================\n# # MODEL TRAINING\n# # =============================================================================\n\n# class HybridModelTrainer:\n#     \"\"\"Train multiple models on different feature sets\"\"\"\n    \n#     def __init__(self, config: MemoryOptimizedHybridConfiguration):\n#         self.config = config\n#         self.models = {}\n#         self.oof_predictions = {}\n    \n#     def train_models(self, X_train: np.ndarray, X_test: np.ndarray, y_train: np.ndarray, \n#                     feature_sets: Dict[str, Dict[str, np.ndarray]], \n#                     train_indices: np.ndarray) -> Dict[str, Any]:\n#         \"\"\"Train models on original and reduced feature sets\"\"\"\n        \n#         # Create train/validation split\n#         X_train_split, X_val_split, y_train_split, y_val_split, idx_train, idx_val = train_test_split(\n#             X_train, y_train, train_indices,\n#             test_size=self.config.validation_size, \n#             random_state=self.config.random_state\n#         )\n        \n#         # Initialize OOF predictions array\n#         oof_predictions_full = np.zeros(len(train_indices))\n        \n#         # 1. XGBoost on original features\n#         print(\"\\nTraining XGBoost on original features...\")\n#         xgb_model = XGBRegressor(**self.config.xgb_params)\n        \n#         xgb_model.fit(\n#             X_train_split, y_train_split,\n#             eval_set=[(X_val_split, y_val_split)],\n#             early_stopping_rounds=30,\n#             verbose=False\n#         )\n        \n#         val_pred_xgb = xgb_model.predict(X_val_split)\n#         score_xgb = pearsonr(y_val_split, val_pred_xgb)[0]\n#         print(f\"  Original features score: {score_xgb:.4f}\")\n        \n#         # Store OOF predictions\n#         oof_predictions_full[idx_val] = val_pred_xgb\n        \n#         self.models['xgb_original'] = {\n#             'model': xgb_model,\n#             'score': score_xgb,\n#             'predictions': xgb_model.predict(X_test),\n#             'feature_type': 'original'\n#         }\n        \n#         # 2. LightGBM on original features\n#         print(\"\\nTraining LightGBM on original features...\")\n#         lgb_train = lgb.Dataset(X_train_split, label=y_train_split)\n#         lgb_val = lgb.Dataset(X_val_split, label=y_val_split, reference=lgb_train)\n        \n#         lgb_model = lgb.train(\n#             self.config.lgb_params,\n#             lgb_train,\n#             valid_sets=[lgb_val],\n#             num_boost_round=500,\n#             callbacks=[lgb.early_stopping(30), lgb.log_evaluation(0)]\n#         )\n        \n#         val_pred_lgb = lgb_model.predict(X_val_split, num_iteration=lgb_model.best_iteration)\n#         score_lgb = pearsonr(y_val_split, val_pred_lgb)[0]\n#         print(f\"  LightGBM score: {score_lgb:.4f}\")\n        \n#         self.models['lgb_original'] = {\n#             'model': lgb_model,\n#             'score': score_lgb,\n#             'predictions': lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration),\n#             'feature_type': 'original'\n#         }\n        \n#         # 3. Models on dimensionality reductions\n#         for name, feature_data in feature_sets.items():\n#             print(f\"\\nTraining XGBoost on {name} features...\")\n            \n#             X_train_reduced = feature_data['train']\n#             X_test_reduced = feature_data['test']\n            \n#             # Get splits using same indices\n#             X_train_r = X_train_reduced[idx_train]\n#             X_val_r = X_train_reduced[idx_val]\n            \n#             xgb_reduced = XGBRegressor(**self.config.xgb_params)\n            \n#             xgb_reduced.fit(\n#                 X_train_r, y_train_split,\n#                 eval_set=[(X_val_r, y_val_split)],\n#                 early_stopping_rounds=30,\n#                 verbose=False\n#             )\n            \n#             val_pred_reduced = xgb_reduced.predict(X_val_r)\n#             score = pearsonr(y_val_split, val_pred_reduced)[0]\n#             print(f\"  {name} score: {score:.4f}\")\n            \n#             self.models[f'xgb_{name}'] = {\n#                 'model': xgb_reduced,\n#                 'score': score,\n#                 'predictions': xgb_reduced.predict(X_test_reduced),\n#                 'feature_type': name\n#             }\n        \n#         # 4. Create ensemble\n#         print(\"\\nCreating ensemble predictions...\")\n#         sorted_models = sorted(self.models.items(), key=lambda x: x[1]['score'], reverse=True)\n        \n#         # Use top 3 models\n#         top_models = sorted_models[:3]\n        \n#         # Simple average ensemble\n#         ensemble_predictions = np.zeros(len(X_test))\n#         ensemble_oof = np.zeros(len(idx_val))\n        \n#         print(\"Ensemble composition:\")\n#         for name, model_data in top_models:\n#             ensemble_predictions += model_data['predictions'] / len(top_models)\n#             print(f\"  {name}: score = {model_data['score']:.4f}\")\n            \n#             # Re-predict validation for ensemble OOF\n#             if 'xgb' in name:\n#                 if name == 'xgb_original':\n#                     val_pred = model_data['model'].predict(X_val_split)\n#                 else:\n#                     feature_type = model_data['feature_type']\n#                     X_val_feat = feature_sets[feature_type]['train'][idx_val]\n#                     val_pred = model_data['model'].predict(X_val_feat)\n#                 ensemble_oof += val_pred / len(top_models)\n        \n#         ensemble_score = pearsonr(y_val_split, ensemble_oof)[0]\n#         print(f\"\\nEnsemble validation score: {ensemble_score:.4f}\")\n        \n#         self.models['ensemble'] = {\n#             'predictions': ensemble_predictions,\n#             'score': ensemble_score,\n#             'composition': [name for name, _ in top_models]\n#         }\n        \n#         return {\n#             'models': self.models,\n#             'oof_predictions': oof_predictions_full,\n#             'val_indices': idx_val,\n#             'val_score': ensemble_score\n#         }\n\n# # =============================================================================\n# # MAIN PIPELINE\n# # =============================================================================\n\n# class MemoryOptimizedHybridPipeline:\n#     \"\"\"Memory-Optimized Hybrid Pipeline with comprehensive tracking\"\"\"\n    \n#     def __init__(self, config: Optional[MemoryOptimizedHybridConfiguration] = None):\n#         self.config = config or MemoryOptimizedHybridConfiguration()\n#         self.data_processor = MemoryEfficientDataProcessor(self.config)\n#         self.feature_selector = MemoryEfficientFeatureSelector(self.config)\n#         self.dim_reducer = DimensionalityReducer(self.config)\n#         self.model_trainer = HybridModelTrainer(self.config)\n        \n#     def load_and_prepare_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n#         \"\"\"Load and prepare data\"\"\"\n#         print(\"Loading data...\")\n        \n#         train_df = pd.read_parquet(global_config.train_path)\n#         test_df = pd.read_parquet(global_config.test_path)\n#         sample_submission = pd.read_csv(global_config.sample_sub_path)\n        \n#         print(f\"Train shape: {train_df.shape}\")\n#         print(f\"Test shape: {test_df.shape}\")\n#         print(f\"Memory usage: {train_df.memory_usage().sum() / 1e9:.2f} GB\")\n        \n#         return train_df, test_df, sample_submission\n    \n#     def run(self) -> float:\n#         \"\"\"Execute the complete pipeline\"\"\"\n#         print(\"\\nStarting Memory-Optimized Hybrid Pipeline\")\n#         print(\"=\"*80)\n        \n#         # Update status\n#         global_config.update_model_status(self.config.model_name, 'running')\n        \n#         try:\n#             # Load data\n#             train_df, test_df, sample_submission = self.load_and_prepare_data()\n            \n#             # Get feature columns\n#             feature_cols = [col for col in train_df.columns if col not in ['label', 'timestamp']]\n#             print(f\"Initial features: {len(feature_cols)}\")\n            \n#             # Add original indices for tracking\n#             train_df['original_index'] = range(len(train_df))\n#             test_df['original_index'] = range(len(test_df))\n            \n#             # Memory-efficient data cleaning\n#             print(\"\\nCleaning data in chunks...\")\n#             X_train_clean = self.data_processor.clean_financial_data_chunked(train_df, feature_cols)\n#             X_test_clean = self.data_processor.clean_financial_data_chunked(test_df, feature_cols)\n#             y_train = train_df['label'].values\n#             train_indices = train_df['original_index'].values\n            \n#             gc.collect()\n            \n#             # Lightweight feature engineering\n#             print(\"\\nApplying lightweight feature engineering...\")\n#             X_train_eng = self.data_processor.create_lightweight_features(X_train_clean, n_interactions=10)\n#             X_test_eng = self.data_processor.create_lightweight_features(X_test_clean, n_interactions=10)\n#             print(f\"Features after engineering: {X_train_eng.shape[1]}\")\n            \n#             # Clean up\n#             del X_train_clean, X_test_clean\n#             gc.collect()\n            \n#             # Feature selection\n#             selected_features, feature_scores = self.feature_selector.select_features(\n#                 X_train_eng, y_train, n_features=self.config.n_top_features\n#             )\n            \n#             X_train_selected = X_train_eng[selected_features]\n#             X_test_selected = X_test_eng[selected_features]\n            \n#             # Save selected features\n#             with open(self.config.feature_list_path, 'w') as f:\n#                 json.dump(selected_features, f, indent=2)\n            \n#             global_config.register_model_output(\n#                 self.config.model_name,\n#                 'selected_features',\n#                 self.config.feature_list_path,\n#                 'features',\n#                 metadata={'n_features': len(selected_features)}\n#             )\n            \n#             # Clean up\n#             del X_train_eng, X_test_eng\n#             gc.collect()\n            \n#             # Sample for training if needed\n#             if len(X_train_selected) > self.config.training_sample:\n#                 print(f\"\\nSampling {self.config.training_sample:,} rows for training...\")\n#                 sample_indices = np.random.choice(len(X_train_selected), self.config.training_sample, replace=False)\n#                 X_train_sample = X_train_selected.iloc[sample_indices].values\n#                 y_train_sample = y_train[sample_indices]\n#                 train_indices_sample = train_indices[sample_indices]\n#             else:\n#                 X_train_sample = X_train_selected.values\n#                 y_train_sample = y_train\n#                 train_indices_sample = train_indices\n            \n#             X_test_array = X_test_selected.values\n            \n#             # Clean up\n#             del X_train_selected, X_test_selected\n#             gc.collect()\n            \n#             # Apply dimensionality reduction\n#             reduced_features = self.dim_reducer.fit_transform(\n#                 X_train_sample, X_test_array, y_train_sample\n#             )\n            \n#             # Save dimensionality reduction models\n#             with open(self.config.dimensionality_reduction_path, 'wb') as f:\n#                 pickle.dump(self.dim_reducer.models, f)\n            \n#             # Train models\n#             training_results = self.model_trainer.train_models(\n#                 X_train_sample, X_test_array, y_train_sample, \n#                 reduced_features, train_indices_sample\n#             )\n            \n#             # Save all outputs\n#             self.save_outputs(\n#                 training_results=training_results,\n#                 train_indices=train_indices_sample,\n#                 y_train=y_train_sample,\n#                 sample_submission=sample_submission,\n#                 selected_features=selected_features,\n#                 feature_scores=feature_scores,\n#                 reduced_features=reduced_features\n#             )\n            \n#             # Update status\n#             final_score = training_results['val_score']\n#             global_config.update_model_status(\n#                 self.config.model_name, \n#                 'completed', \n#                 score=final_score\n#             )\n            \n#             print(f\"\\n✅ Memory-Optimized Hybrid pipeline completed successfully\")\n#             print(f\"  Final Score: {final_score:.6f}\")\n            \n#             return final_score\n            \n#         except Exception as e:\n#             error_msg = str(e)\n#             global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n#             print(f\"\\n❌ Memory-Optimized Hybrid pipeline failed: {error_msg}\")\n#             raise\n            \n#         finally:\n#             aggressive_memory_cleanup()\n#             print(\"\\n\" + \"=\"*80)\n#             print(\"Current Execution Status:\")\n#             print(global_config.get_execution_summary())\n#             print(\"=\"*80)\n    \n#     def save_outputs(self, training_results, train_indices, y_train, sample_submission,\n#                     selected_features, feature_scores, reduced_features):\n#         \"\"\"Save all outputs for meta-learning\"\"\"\n        \n#         # Save submission\n#         submission = sample_submission.copy()\n#         submission['prediction'] = training_results['models']['ensemble']['predictions']\n#         submission.to_csv(self.config.submission_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'submission',\n#             self.config.submission_path,\n#             'submission',\n#             metadata={\n#                 'score': float(training_results['val_score']),\n#                 'ensemble_composition': training_results['models']['ensemble']['composition']\n#             }\n#         )\n        \n#         # Save OOF predictions\n#         oof_df = pd.DataFrame({\n#             'row_id': train_indices,\n#             'true_label': y_train,\n#             'prediction': training_results['oof_predictions']\n#         })\n#         oof_df.to_csv(self.config.oof_predictions_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'oof_predictions',\n#             self.config.oof_predictions_path,\n#             'oof_predictions',\n#             metadata={'n_samples': len(oof_df)}\n#         )\n        \n#         # Save model performance\n#         performance_data = {\n#             'models': {},\n#             'ensemble_score': float(training_results['val_score']),\n#             'feature_selection': {\n#                 'n_selected': len(selected_features),\n#                 'top_features': selected_features[:20]\n#             },\n#             'dimensionality_reduction': {\n#                 name: {'n_components': data['n_components']}\n#                 for name, data in reduced_features.items()\n#             }\n#         }\n        \n#         for model_name, model_data in training_results['models'].items():\n#             if model_name != 'ensemble':\n#                 performance_data['models'][model_name] = {\n#                     'score': float(model_data['score']),\n#                     'feature_type': model_data.get('feature_type', 'original')\n#                 }\n        \n#         with open(self.config.model_performance_path, 'w') as f:\n#             json.dump(performance_data, f, indent=2)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'model_performance',\n#             self.config.model_performance_path,\n#             'analysis',\n#             metadata=performance_data\n#         )\n        \n#         # Save feature importance\n#         feature_importance_df = pd.DataFrame({\n#             'feature': selected_features,\n#             'importance': feature_scores[np.argsort(feature_scores)[-len(selected_features):]]\n#         }).sort_values('importance', ascending=False)\n        \n#         feature_importance_df.to_csv(self.config.feature_importance_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'feature_importance',\n#             self.config.feature_importance_path,\n#             'analysis',\n#             metadata={'top_features': feature_importance_df.head(10)['feature'].tolist()}\n#         )\n        \n#         # Save models\n#         with open(self.config.models_path, 'wb') as f:\n#             pickle.dump(training_results['models'], f)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'models',\n#             self.config.models_path,\n#             'model'\n#         )\n        \n#         # Save configuration\n#         self.config.save()\n        \n#         # Create meta features\n#         meta_features = self.create_meta_features(\n#             training_results['oof_predictions'],\n#             y_train,\n#             training_results['models']\n#         )\n#         meta_features['row_id'] = train_indices\n        \n#         cols = ['row_id'] + [col for col in meta_features.columns if col != 'row_id']\n#         meta_features = meta_features[cols]\n        \n#         meta_features.to_csv(self.config.meta_features_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'meta_features',\n#             self.config.meta_features_path,\n#             'features',\n#             metadata={'n_features': len(meta_features.columns) - 1}\n#         )\n        \n#         print(f\"\\nAll outputs saved for {self.config.model_name}\")\n#         print(\"\\nModel Performance Summary:\")\n#         for name, data in sorted(performance_data['models'].items(), \n#                                key=lambda x: x[1]['score'], reverse=True):\n#             print(f\"  {name}: {data['score']:.4f} ({data['feature_type']})\")\n    \n#     def create_meta_features(self, predictions, y_true, models_dict):\n#         \"\"\"Create meta-features for meta-learning\"\"\"\n#         meta_features = {}\n        \n#         # Basic prediction features\n#         meta_features['prediction'] = predictions\n#         meta_features['error'] = predictions - y_true\n#         meta_features['abs_error'] = np.abs(predictions - y_true)\n#         meta_features['squared_error'] = (predictions - y_true) ** 2\n        \n#         # Model diversity features\n#         model_predictions = []\n#         for name, model_data in models_dict.items():\n#             if 'predictions' in model_data and name != 'ensemble':\n#                 model_predictions.append(model_data['predictions'])\n        \n#         if len(model_predictions) > 1:\n#             model_predictions_array = np.column_stack(model_predictions)\n#             meta_features['model_std'] = np.std(model_predictions_array, axis=1)\n#             meta_features['model_range'] = np.ptp(model_predictions_array, axis=1)\n        \n#         # Prediction confidence\n#         pred_percentile = np.percentile(predictions, [10, 25, 50, 75, 90])\n#         for i, p in enumerate([10, 25, 50, 75, 90]):\n#             meta_features[f'above_p{p}'] = (predictions > pred_percentile[i]).astype(int)\n        \n#         # Normalized prediction\n#         pred_mean = np.mean(predictions)\n#         pred_std = np.std(predictions)\n#         meta_features['prediction_zscore'] = (predictions - pred_mean) / (pred_std + 1e-8)\n        \n#         return pd.DataFrame(meta_features)\n\n# # =============================================================================\n# # MAIN EXECUTION\n# # =============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\\n🧠 Running Memory-Optimized Hybrid Pipeline\")\n#     print(\"-\"*80)\n    \n#     # Clean memory\n#     aggressive_memory_cleanup()\n    \n#     # Create configuration\n#     config = MemoryOptimizedHybridConfiguration(\n#         chunk_size=50000,\n#         training_sample=350000,\n#         n_top_features=200,\n#         n_pca_components=40,\n#         n_svd_components=30,\n#         n_umap_components=50\n#     )\n    \n#     # Create and run pipeline\n#     pipeline = MemoryOptimizedHybridPipeline(config)\n#     final_score = pipeline.run()","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # CROSS-VALIDATION STRATEGY ENSEMBLE PIPELINE - FULLY ENCAPSULATED IMPLEMENTATION\n# # !/usr/bin/env python\n# # -*- coding: utf-8 -*-\n# \"\"\"\n# DRW Crypto Market Prediction - Cross-Validation Strategy Ensemble Pipeline\n# Explores multiple validation strategies and creates an intelligent ensemble\n# \"\"\"\n\n# import subprocess\n# import sys\n# import os\n# import gc\n# import warnings\n# import json\n# import pickle\n# import pandas as pd\n# import numpy as np\n# from scipy.stats import pearsonr\n# from typing import List, Dict, Tuple, Optional, Any, Union\n# from pathlib import Path\n# from dataclasses import dataclass, field\n# from datetime import datetime\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# from sklearn.model_selection import KFold\n\n# warnings.filterwarnings('ignore')\n\n# # Check that setup cell was run\n# try:\n#     global_config\n#     print(\"✓ Global configuration found\")\n# except NameError:\n#     raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# # Install required packages\n# print(\"Installing packages for Cross-Validation Strategy Ensemble pipeline...\")\n# packages_to_install = [\n#     'xgboost==2.0.3',\n#     'matplotlib',\n#     'seaborn'\n# ]\n\n# for package in packages_to_install:\n#     try:\n#         subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n#     except:\n#         print(f\"Warning: Could not install {package}\")\n\n# # Import XGBoost after installation\n# from xgboost import XGBRegressor\n\n# # =============================================================================\n# # CONFIGURATION\n# # =============================================================================\n\n# @dataclass\n# class CrossValidationStrategyConfiguration:\n#     \"\"\"Configuration for Cross-Validation Strategy Ensemble Pipeline\"\"\"\n    \n#     # Model identification\n#     model_name: str = \"cv_strategy_ensemble\"\n#     model_directory: str = \"\"\n    \n#     # Selected features (from the code)\n#     selected_features: List[str] = field(default_factory=list)\n    \n#     # XGBoost parameters (from the code)\n#     xgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n#     # Cross-validation parameters\n#     n_splits: int = 5\n#     random_state: int = 42\n#     early_stopping_rounds: int = 25\n#     verbose_eval: int = 200\n    \n#     # Validation strategies to test\n#     validation_strategies: List[str] = field(default_factory=list)\n    \n#     # Ensemble parameters\n#     use_best_strategy_only: bool = False  # If True, use only best strategy; if False, ensemble all\n#     ensemble_weights_method: str = \"performance\"  # \"performance\", \"equal\", or \"inverse_variance\"\n#     min_strategy_score: float = 0.0  # Minimum score to include strategy in ensemble\n    \n#     def __post_init__(self):\n#         \"\"\"Initialize configuration and register model\"\"\"\n#         if not self.model_directory:\n#             self.model_directory = os.path.join(global_config.base_dir, \"cv_strategy_ensemble\")\n        \n#         # Register model with global configuration\n#         global_config.register_model(self.model_name, self.model_directory)\n        \n#         # Set default features if not provided\n#         if not self.selected_features:\n#             self.selected_features = [\n#                 \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n#                 \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n#                 \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n#             ]\n        \n#         # Default XGBoost parameters (from the code)\n#         if not self.xgb_params:\n#             self.xgb_params = {\n#                 \"tree_method\": \"hist\",  # Changed from gpu_hist for compatibility\n#                 \"device\": \"cpu\",  # Explicitly set to CPU\n#                 \"colsample_bylevel\": 0.4778015829774066,\n#                 \"colsample_bynode\": 0.362764358742407,\n#                 \"colsample_bytree\": 0.7107423488010493,\n#                 \"gamma\": 1.7094857725240398,\n#                 \"learning_rate\": 0.02213323588455387,\n#                 \"max_depth\": 20,\n#                 \"max_leaves\": 12,\n#                 \"min_child_weight\": 16,\n#                 \"n_estimators\": 1667,\n#                 \"n_jobs\": -1,\n#                 \"random_state\": self.random_state,\n#                 \"reg_alpha\": 39.352415706891264,\n#                 \"reg_lambda\": 75.44843704068275,\n#                 \"subsample\": 0.06566669853471274,\n#                 \"verbosity\": 0\n#             }\n        \n#         # Default validation strategies\n#         if not self.validation_strategies:\n#             self.validation_strategies = [\n#                 \"kfold_no_shuffle\",\n#                 \"kfold_shuffle\",\n#                 \"walk_forward_expanding\",\n#                 \"walk_forward_sliding\"\n#             ]\n        \n#         # Output paths\n#         self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n#         self.models_path = os.path.join(self.model_directory, \"models.pkl\")\n#         self.config_path = os.path.join(self.model_directory, \"config.json\")\n        \n#         # Analysis paths\n#         self.strategy_comparison_path = os.path.join(self.model_directory, \"strategy_comparison.csv\")\n#         self.visualization_path = os.path.join(self.model_directory, \"validation_strategies.png\")\n#         self.detailed_results_path = os.path.join(self.model_directory, \"detailed_results.json\")\n        \n#         # Meta-learning data paths\n#         self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n#         self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n#         self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n#         self.fold_predictions_path = os.path.join(self.model_directory, \"fold_predictions.pkl\")\n        \n#         # Ensure directories exist\n#         Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n    \n#     def save(self):\n#         \"\"\"Save configuration to JSON file\"\"\"\n#         config_dict = {\n#             'model_name': self.model_name,\n#             'selected_features': self.selected_features,\n#             'xgb_params': self.xgb_params,\n#             'n_splits': self.n_splits,\n#             'random_state': self.random_state,\n#             'validation_strategies': self.validation_strategies,\n#             'use_best_strategy_only': self.use_best_strategy_only,\n#             'ensemble_weights_method': self.ensemble_weights_method\n#         }\n#         with open(self.config_path, 'w') as f:\n#             json.dump(config_dict, f, indent=2)\n\n# # =============================================================================\n# # VALIDATION STRATEGY IMPLEMENTATIONS\n# # =============================================================================\n\n# class ValidationStrategyManager:\n#     \"\"\"Manages different cross-validation strategies\"\"\"\n    \n#     def __init__(self, config: CrossValidationStrategyConfiguration):\n#         self.config = config\n#         self.strategies = {\n#             \"kfold_no_shuffle\": self._kfold_no_shuffle,\n#             \"kfold_shuffle\": self._kfold_shuffle,\n#             \"walk_forward_expanding\": self._walk_forward_expanding,\n#             \"walk_forward_sliding\": self._walk_forward_sliding\n#         }\n    \n#     def get_splits(self, df: pd.DataFrame, strategy: str) -> List[Tuple[np.ndarray, np.ndarray]]:\n#         \"\"\"Get train/validation splits for a given strategy\"\"\"\n#         if strategy not in self.strategies:\n#             raise ValueError(f\"Unknown strategy: {strategy}\")\n        \n#         return self.strategies[strategy](df)\n    \n#     def _kfold_no_shuffle(self, df: pd.DataFrame) -> List[Tuple[np.ndarray, np.ndarray]]:\n#         \"\"\"K-fold without shuffling (preserves temporal order)\"\"\"\n#         kf = KFold(n_splits=self.config.n_splits, shuffle=False)\n#         return list(kf.split(df))\n    \n#     def _kfold_shuffle(self, df: pd.DataFrame) -> List[Tuple[np.ndarray, np.ndarray]]:\n#         \"\"\"K-fold with shuffling (breaks temporal order)\"\"\"\n#         kf = KFold(n_splits=self.config.n_splits, shuffle=True, random_state=self.config.random_state)\n#         return list(kf.split(df))\n    \n#     def _walk_forward_expanding(self, df: pd.DataFrame) -> List[Tuple[np.ndarray, np.ndarray]]:\n#         \"\"\"Walk-forward validation with expanding window\"\"\"\n#         splits = []\n#         range_length = len(df) // (self.config.n_splits + 1)\n        \n#         for i in range(self.config.n_splits):\n#             start_idx = range_length * (i + 1)\n#             end_idx = start_idx + range_length\n            \n#             train_idx = np.arange(0, start_idx)\n#             valid_idx = np.arange(start_idx, min(end_idx, len(df)))\n            \n#             splits.append((train_idx, valid_idx))\n        \n#         return splits\n    \n#     def _walk_forward_sliding(self, df: pd.DataFrame) -> List[Tuple[np.ndarray, np.ndarray]]:\n#         \"\"\"Walk-forward validation with sliding window\"\"\"\n#         splits = []\n#         range_length = len(df) // (self.config.n_splits + 1)\n        \n#         for i in range(self.config.n_splits):\n#             start_idx = range_length * (i + 1)\n#             end_idx = start_idx + range_length\n            \n#             train_start = max(0, start_idx - range_length)\n#             train_idx = np.arange(train_start, start_idx)\n#             valid_idx = np.arange(start_idx, min(end_idx, len(df)))\n            \n#             splits.append((train_idx, valid_idx))\n        \n#         return splits\n\n# # =============================================================================\n# # MODEL TRAINER\n# # =============================================================================\n\n# class StrategyModelTrainer:\n#     \"\"\"Trains models using different validation strategies\"\"\"\n    \n#     def __init__(self, config: CrossValidationStrategyConfiguration):\n#         self.config = config\n#         self.strategy_manager = ValidationStrategyManager(config)\n#         self.results = {}\n    \n#     def train_strategy(self, df_train: pd.DataFrame, df_test: pd.DataFrame, \n#                       strategy: str) -> Dict[str, Any]:\n#         \"\"\"Train model using a specific validation strategy\"\"\"\n#         print(f\"\\n{'='*60}\")\n#         print(f\"Training with strategy: {strategy}\")\n#         print('='*60)\n        \n#         # Get splits for this strategy\n#         splits = self.strategy_manager.get_splits(df_train, strategy)\n        \n#         # Initialize storage\n#         oof_preds = np.zeros(len(df_train))\n#         test_preds = np.zeros(len(df_test))\n#         fold_scores = []\n#         fold_models = []\n#         cv_stats = []\n        \n#         # Train on each fold\n#         for fold_idx, (train_idx, valid_idx) in enumerate(splits):\n#             print(f\"\\n### Fold {fold_idx + 1}/{len(splits)} ###\")\n            \n#             # Prepare data\n#             X_train = df_train.iloc[train_idx][self.config.selected_features]\n#             y_train = df_train.iloc[train_idx][\"label\"]\n#             X_valid = df_train.iloc[valid_idx][self.config.selected_features]\n#             y_valid = df_train.iloc[valid_idx][\"label\"]\n#             X_test = df_test[self.config.selected_features]\n            \n#             # Skip if not enough data\n#             if len(X_train) < 100 or len(X_valid) < 50:\n#                 print(f\"  Skipping fold {fold_idx + 1} - insufficient data\")\n#                 continue\n            \n#             # Train model\n#             model = XGBRegressor(**self.config.xgb_params)\n            \n#             model.fit(\n#                 X_train, y_train,\n#                 eval_set=[(X_valid, y_valid)],\n#                 early_stopping_rounds=self.config.early_stopping_rounds,\n#                 verbose=self.config.verbose_eval\n#             )\n            \n#             # Make predictions\n#             valid_pred = model.predict(X_valid)\n#             test_pred = model.predict(X_test)\n            \n#             oof_preds[valid_idx] = valid_pred\n#             test_preds += test_pred / len(splits)\n            \n#             # Calculate scores\n#             train_score = pearsonr(y_train, model.predict(X_train))[0]\n#             valid_score = pearsonr(y_valid, valid_pred)[0]\n            \n#             fold_scores.append(valid_score)\n#             fold_models.append(model)\n            \n#             cv_stats.append({\n#                 'fold': fold_idx + 1,\n#                 'train_score': train_score,\n#                 'valid_score': valid_score,\n#                 'train_size': len(train_idx),\n#                 'valid_size': len(valid_idx),\n#                 'n_iterations': model.best_iteration if hasattr(model, 'best_iteration') else self.config.xgb_params['n_estimators']\n#             })\n            \n#             print(f\"  Train score: {train_score:.4f}, Valid score: {valid_score:.4f}\")\n        \n#         # Calculate overall score (only on folds that were actually predicted)\n#         mask = oof_preds != 0\n#         if np.sum(mask) > 0:\n#             overall_score = pearsonr(df_train.loc[mask, \"label\"], oof_preds[mask])[0]\n#         else:\n#             overall_score = 0.0\n        \n#         print(f\"\\nOverall OOF score for {strategy}: {overall_score:.4f}\")\n        \n#         return {\n#             'strategy': strategy,\n#             'oof_predictions': oof_preds,\n#             'test_predictions': test_preds,\n#             'overall_score': overall_score,\n#             'fold_scores': fold_scores,\n#             'cv_stats': cv_stats,\n#             'models': fold_models,\n#             'splits': splits\n#         }\n    \n#     def train_all_strategies(self, df_train: pd.DataFrame, df_test: pd.DataFrame) -> Dict[str, Dict[str, Any]]:\n#         \"\"\"Train models using all configured strategies\"\"\"\n#         results = {}\n        \n#         for strategy in self.config.validation_strategies:\n#             try:\n#                 result = self.train_strategy(df_train, df_test, strategy)\n#                 results[strategy] = result\n#             except Exception as e:\n#                 print(f\"\\nError training strategy {strategy}: {e}\")\n#                 continue\n        \n#         return results\n\n# # =============================================================================\n# # ENSEMBLE BUILDER\n# # =============================================================================\n\n# class StrategyEnsembleBuilder:\n#     \"\"\"Builds ensemble from different validation strategies\"\"\"\n    \n#     def __init__(self, config: CrossValidationStrategyConfiguration):\n#         self.config = config\n    \n#     def create_ensemble(self, strategy_results: Dict[str, Dict[str, Any]], \n#                        df_train: pd.DataFrame, df_test: pd.DataFrame) -> Dict[str, Any]:\n#         \"\"\"Create ensemble from strategy results\"\"\"\n#         print(\"\\n\" + \"=\"*60)\n#         print(\"Creating ensemble from validation strategies\")\n#         print(\"=\"*60)\n        \n#         # Extract scores and predictions\n#         strategy_scores = {\n#             strategy: result['overall_score'] \n#             for strategy, result in strategy_results.items()\n#         }\n        \n#         # Filter strategies by minimum score\n#         valid_strategies = {\n#             strategy: score \n#             for strategy, score in strategy_scores.items() \n#             if score >= self.config.min_strategy_score\n#         }\n        \n#         if not valid_strategies:\n#             print(\"No strategies met minimum score requirement!\")\n#             # Use all strategies as fallback\n#             valid_strategies = strategy_scores\n        \n#         print(\"\\nStrategy performance:\")\n#         for strategy, score in sorted(valid_strategies.items(), key=lambda x: x[1], reverse=True):\n#             print(f\"  {strategy}: {score:.4f}\")\n        \n#         # Determine ensemble approach\n#         if self.config.use_best_strategy_only:\n#             # Use only the best strategy\n#             best_strategy = max(valid_strategies.items(), key=lambda x: x[1])[0]\n#             print(f\"\\nUsing best strategy only: {best_strategy}\")\n            \n#             ensemble_oof = strategy_results[best_strategy]['oof_predictions']\n#             ensemble_test = strategy_results[best_strategy]['test_predictions']\n#             weights = {best_strategy: 1.0}\n            \n#         else:\n#             # Ensemble multiple strategies\n#             weights = self._calculate_weights(valid_strategies, strategy_results)\n            \n#             print(\"\\nEnsemble weights:\")\n#             for strategy, weight in weights.items():\n#                 print(f\"  {strategy}: {weight:.3f}\")\n            \n#             # Create weighted ensemble\n#             ensemble_oof = np.zeros(len(df_train))\n#             ensemble_test = np.zeros(len(df_test))\n            \n#             for strategy, weight in weights.items():\n#                 ensemble_oof += weight * strategy_results[strategy]['oof_predictions']\n#                 ensemble_test += weight * strategy_results[strategy]['test_predictions']\n        \n#         # Calculate ensemble score\n#         mask = ensemble_oof != 0\n#         if np.sum(mask) > 0:\n#             ensemble_score = pearsonr(df_train.loc[mask, \"label\"], ensemble_oof[mask])[0]\n#         else:\n#             ensemble_score = 0.0\n        \n#         print(f\"\\nFinal ensemble score: {ensemble_score:.4f}\")\n        \n#         return {\n#             'oof_predictions': ensemble_oof,\n#             'test_predictions': ensemble_test,\n#             'ensemble_score': ensemble_score,\n#             'weights': weights,\n#             'strategy_scores': strategy_scores\n#         }\n    \n#     def _calculate_weights(self, valid_strategies: Dict[str, float], \n#                           strategy_results: Dict[str, Dict[str, Any]]) -> Dict[str, float]:\n#         \"\"\"Calculate ensemble weights based on configuration\"\"\"\n        \n#         if self.config.ensemble_weights_method == \"equal\":\n#             # Equal weights\n#             weights = {strategy: 1.0 / len(valid_strategies) for strategy in valid_strategies}\n            \n#         elif self.config.ensemble_weights_method == \"performance\":\n#             # Weight by performance (correlation)\n#             scores = np.array(list(valid_strategies.values()))\n#             # Ensure all positive\n#             scores = np.maximum(scores, 0.01)\n#             normalized_scores = scores / scores.sum()\n#             weights = dict(zip(valid_strategies.keys(), normalized_scores))\n            \n#         elif self.config.ensemble_weights_method == \"inverse_variance\":\n#             # Weight by inverse of prediction variance\n#             variances = {}\n#             for strategy in valid_strategies:\n#                 preds = strategy_results[strategy]['oof_predictions']\n#                 # Only calculate variance on non-zero predictions\n#                 mask = preds != 0\n#                 if np.sum(mask) > 0:\n#                     variances[strategy] = np.var(preds[mask])\n#                 else:\n#                     variances[strategy] = 1.0\n            \n#             # Inverse variance weights\n#             inv_vars = {k: 1.0 / (v + 1e-6) for k, v in variances.items()}\n#             total = sum(inv_vars.values())\n#             weights = {k: v / total for k, v in inv_vars.items()}\n            \n#         else:\n#             # Default to equal weights\n#             weights = {strategy: 1.0 / len(valid_strategies) for strategy in valid_strategies}\n        \n#         return weights\n\n# # =============================================================================\n# # VISUALIZATION\n# # =============================================================================\n\n# class StrategyVisualizer:\n#     \"\"\"Creates visualizations for validation strategies\"\"\"\n    \n#     def __init__(self, config: CrossValidationStrategyConfiguration):\n#         self.config = config\n        \n#         # Define colors\n#         self.colors = [\"#068D9D\", \"#53599A\", \"#607BB0\", \"#6D9DC5\", \"#77BECF\", \"#80DED9\", \"#AEECEF\"]\n#         plt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=True)\n    \n#     def create_comprehensive_visualization(self, strategy_results: Dict[str, Dict[str, Any]], \n#                                          df_train: pd.DataFrame, output_path: str):\n#         \"\"\"Create comprehensive visualization of all strategies\"\"\"\n#         n_strategies = len(strategy_results)\n        \n#         fig = plt.figure(figsize=(20, 12))\n#         gs = fig.add_gridspec(3, 3, hspace=0.3, wspace=0.3)\n        \n#         # 1. Strategy comparison bar plot\n#         ax1 = fig.add_subplot(gs[0, :2])\n#         strategies = list(strategy_results.keys())\n#         scores = [strategy_results[s]['overall_score'] for s in strategies]\n        \n#         bars = ax1.bar(strategies, scores, color=self.colors[:n_strategies])\n#         ax1.set_ylabel('Pearson Correlation')\n#         ax1.set_title('Validation Strategy Performance Comparison')\n#         ax1.set_ylim(0, max(scores) * 1.2)\n        \n#         # Add value labels on bars\n#         for bar, score in zip(bars, scores):\n#             height = bar.get_height()\n#             ax1.text(bar.get_x() + bar.get_width()/2., height,\n#                     f'{score:.3f}', ha='center', va='bottom')\n        \n#         # 2. Fold consistency heatmap\n#         ax2 = fig.add_subplot(gs[0, 2])\n#         fold_data = []\n#         strategy_names = []\n        \n#         for strategy, result in strategy_results.items():\n#             if result['fold_scores']:\n#                 fold_data.append(result['fold_scores'])\n#                 strategy_names.append(strategy)\n        \n#         if fold_data:\n#             # Pad with NaN if different lengths\n#             max_folds = max(len(scores) for scores in fold_data)\n#             padded_data = []\n#             for scores in fold_data:\n#                 padded = scores + [np.nan] * (max_folds - len(scores))\n#                 padded_data.append(padded)\n            \n#             im = ax2.imshow(padded_data, aspect='auto', cmap='coolwarm')\n#             ax2.set_yticks(range(len(strategy_names)))\n#             ax2.set_yticklabels(strategy_names)\n#             ax2.set_xlabel('Fold')\n#             ax2.set_title('Fold Scores Heatmap')\n#             plt.colorbar(im, ax=ax2)\n        \n#         # 3. Train vs Valid scatter for each strategy\n#         for idx, (strategy, result) in enumerate(strategy_results.items()):\n#             if idx < 4:  # Only plot first 4 strategies\n#                 row = 1 + idx // 2\n#                 col = idx % 2\n#                 ax = fig.add_subplot(gs[row, col])\n                \n#                 cv_stats = pd.DataFrame(result['cv_stats'])\n#                 if not cv_stats.empty:\n#                     ax.scatter(cv_stats['train_score'], cv_stats['valid_score'], \n#                              s=100, alpha=0.6, color=self.colors[idx])\n                    \n#                     # Add diagonal line\n#                     min_val = min(cv_stats['train_score'].min(), cv_stats['valid_score'].min())\n#                     max_val = max(cv_stats['train_score'].max(), cv_stats['valid_score'].max())\n#                     ax.plot([min_val, max_val], [min_val, max_val], 'k--', alpha=0.5)\n                    \n#                     ax.set_xlabel('Train Score')\n#                     ax.set_ylabel('Valid Score')\n#                     ax.set_title(f'{strategy} - Train vs Valid')\n#                     ax.grid(True, alpha=0.3)\n        \n#         # 4. Prediction distribution\n#         ax4 = fig.add_subplot(gs[2, :])\n#         for idx, (strategy, result) in enumerate(strategy_results.items()):\n#             oof_preds = result['oof_predictions']\n#             mask = oof_preds != 0\n#             if np.sum(mask) > 0:\n#                 ax4.hist(oof_preds[mask], bins=50, alpha=0.5, \n#                         label=f'{strategy} ({result[\"overall_score\"]:.3f})',\n#                         color=self.colors[idx % len(self.colors)])\n        \n#         ax4.set_xlabel('Prediction Value')\n#         ax4.set_ylabel('Count')\n#         ax4.set_title('OOF Prediction Distributions by Strategy')\n#         ax4.legend()\n        \n#         plt.suptitle('Cross-Validation Strategy Analysis', fontsize=16)\n#         plt.tight_layout()\n#         plt.savefig(output_path, dpi=150, bbox_inches='tight')\n#         plt.close()\n        \n#         print(f\"\\nVisualization saved to: {output_path}\")\n\n# # =============================================================================\n# # MAIN PIPELINE\n# # =============================================================================\n\n# class CrossValidationStrategyPipeline:\n#     \"\"\"Main pipeline for cross-validation strategy ensemble\"\"\"\n    \n#     def __init__(self, config: Optional[CrossValidationStrategyConfiguration] = None):\n#         self.config = config or CrossValidationStrategyConfiguration()\n#         self.trainer = StrategyModelTrainer(self.config)\n#         self.ensemble_builder = StrategyEnsembleBuilder(self.config)\n#         self.visualizer = StrategyVisualizer(self.config)\n    \n#     def load_and_prepare_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n#         \"\"\"Load and prepare data\"\"\"\n#         print(\"Loading data...\")\n        \n#         # Load data\n#         train_df = pd.read_parquet(global_config.train_path)\n#         test_df = pd.read_parquet(global_config.test_path)\n#         sample_submission = pd.read_csv(global_config.sample_sub_path)\n        \n#         # Reset index for train to ensure proper indexing\n#         train_df = train_df.reset_index(drop=True)\n        \n#         # Add original indices for tracking\n#         train_df['original_index'] = range(len(train_df))\n#         test_df['original_index'] = range(len(test_df))\n        \n#         # Apply memory optimization\n#         train_df = reduce_memory_usage(train_df, verbose=False)\n#         test_df = reduce_memory_usage(test_df, verbose=False)\n        \n#         # Remove constant features\n#         constant_features = []\n#         for col in train_df.select_dtypes(include=[np.number]).columns:\n#             if train_df[col].nunique(dropna=False) == 1:\n#                 constant_features.append(col)\n        \n#         if constant_features:\n#             print(f\"Found {len(constant_features)} constant features - removing...\")\n#             train_df = train_df.drop(constant_features, axis=1)\n#             test_constant = [col for col in constant_features if col in test_df.columns]\n#             if test_constant:\n#                 test_df = test_df.drop(test_constant, axis=1)\n        \n#         # Ensure all selected features exist\n#         available_features = [f for f in self.config.selected_features if f in train_df.columns]\n#         missing_features = set(self.config.selected_features) - set(available_features)\n        \n#         if missing_features:\n#             print(f\"Warning: {len(missing_features)} features not found: {missing_features}\")\n        \n#         self.config.selected_features = available_features\n        \n#         print(f\"Data loaded - Train: {train_df.shape}, Test: {test_df.shape}\")\n#         print(f\"Using {len(self.config.selected_features)} features\")\n        \n#         return train_df, test_df, sample_submission\n    \n#     def run(self) -> float:\n#         \"\"\"Execute the complete pipeline\"\"\"\n#         print(\"\\nStarting Cross-Validation Strategy Ensemble Pipeline\")\n#         print(\"=\"*80)\n        \n#         # Update status\n#         global_config.update_model_status(self.config.model_name, 'running')\n        \n#         try:\n#             # Load and prepare data\n#             train_df, test_df, sample_submission = self.load_and_prepare_data()\n            \n#             # Train models using all strategies\n#             strategy_results = self.trainer.train_all_strategies(train_df, test_df)\n            \n#             if not strategy_results:\n#                 raise ValueError(\"No strategies produced valid results\")\n            \n#             # Create ensemble\n#             ensemble_result = self.ensemble_builder.create_ensemble(\n#                 strategy_results, train_df, test_df\n#             )\n            \n#             # Create visualizations\n#             self.visualizer.create_comprehensive_visualization(\n#                 strategy_results, train_df, self.config.visualization_path\n#             )\n            \n#             # Save all outputs\n#             self.save_outputs(\n#                 strategy_results=strategy_results,\n#                 ensemble_result=ensemble_result,\n#                 train_df=train_df,\n#                 sample_submission=sample_submission\n#             )\n            \n#             # Update status\n#             final_score = ensemble_result['ensemble_score']\n#             global_config.update_model_status(\n#                 self.config.model_name, \n#                 'completed', \n#                 score=final_score\n#             )\n            \n#             print(f\"\\n✅ Cross-Validation Strategy Ensemble pipeline completed successfully\")\n#             print(f\"  Final Score: {final_score:.6f}\")\n            \n#             return final_score\n            \n#         except Exception as e:\n#             error_msg = str(e)\n#             global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n#             print(f\"\\n❌ Cross-Validation Strategy Ensemble pipeline failed: {error_msg}\")\n#             raise\n            \n#         finally:\n#             aggressive_memory_cleanup()\n#             print(\"\\n\" + \"=\"*80)\n#             print(\"Current Execution Status:\")\n#             print(global_config.get_execution_summary())\n#             print(\"=\"*80)\n    \n#     def save_outputs(self, strategy_results: Dict[str, Dict[str, Any]], \n#                     ensemble_result: Dict[str, Any],\n#                     train_df: pd.DataFrame,\n#                     sample_submission: pd.DataFrame):\n#         \"\"\"Save all outputs for meta-learning\"\"\"\n        \n#         # Save submission\n#         submission = sample_submission.copy()\n#         submission['prediction'] = ensemble_result['test_predictions']\n#         submission.to_csv(self.config.submission_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'submission',\n#             self.config.submission_path,\n#             'submission',\n#             metadata={\n#                 'score': float(ensemble_result['ensemble_score']),\n#                 'n_strategies': len(strategy_results),\n#                 'ensemble_method': self.config.ensemble_weights_method\n#             }\n#         )\n        \n#         # Save OOF predictions\n#         oof_df = pd.DataFrame({\n#             'row_id': train_df['original_index'].values,\n#             'true_label': train_df['label'].values,\n#             'prediction': ensemble_result['oof_predictions']\n#         })\n        \n#         # Add strategy-specific predictions\n#         for strategy, result in strategy_results.items():\n#             oof_df[f'pred_{strategy}'] = result['oof_predictions']\n        \n#         oof_df.to_csv(self.config.oof_predictions_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'oof_predictions',\n#             self.config.oof_predictions_path,\n#             'oof_predictions',\n#             metadata={'n_samples': len(oof_df), 'n_strategies': len(strategy_results)}\n#         )\n        \n#         # Save strategy comparison\n#         comparison_data = []\n#         for strategy, result in strategy_results.items():\n#             stats_df = pd.DataFrame(result['cv_stats'])\n#             comparison_data.append({\n#                 'strategy': strategy,\n#                 'overall_score': result['overall_score'],\n#                 'mean_fold_score': np.mean(result['fold_scores']) if result['fold_scores'] else 0,\n#                 'std_fold_score': np.std(result['fold_scores']) if result['fold_scores'] else 0,\n#                 'n_folds': len(result['fold_scores']),\n#                 'ensemble_weight': ensemble_result['weights'].get(strategy, 0)\n#             })\n        \n#         comparison_df = pd.DataFrame(comparison_data)\n#         comparison_df.to_csv(self.config.strategy_comparison_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'strategy_comparison',\n#             self.config.strategy_comparison_path,\n#             'analysis',\n#             metadata={'best_strategy': comparison_df.loc[comparison_df['overall_score'].idxmax(), 'strategy']}\n#         )\n        \n#         # Save detailed results\n#         detailed_results = {\n#             'strategy_results': {},\n#             'ensemble_result': {\n#                 'score': float(ensemble_result['ensemble_score']),\n#                 'weights': ensemble_result['weights'],\n#                 'strategy_scores': ensemble_result['strategy_scores']\n#             },\n#             'config': {\n#                 'n_splits': self.config.n_splits,\n#                 'ensemble_method': self.config.ensemble_weights_method,\n#                 'use_best_only': self.config.use_best_strategy_only\n#             }\n#         }\n        \n#         for strategy, result in strategy_results.items():\n#             detailed_results['strategy_results'][strategy] = {\n#                 'overall_score': float(result['overall_score']),\n#                 'fold_scores': [float(s) for s in result['fold_scores']],\n#                 'cv_stats': result['cv_stats']\n#             }\n        \n#         with open(self.config.detailed_results_path, 'w') as f:\n#             json.dump(detailed_results, f, indent=2)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'detailed_results',\n#             self.config.detailed_results_path,\n#             'analysis'\n#         )\n        \n#         # Save visualization\n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'visualization',\n#             self.config.visualization_path,\n#             'visualization'\n#         )\n        \n#         # Save models\n#         models_dict = {\n#             strategy: result['models'] \n#             for strategy, result in strategy_results.items()\n#         }\n        \n#         with open(self.config.models_path, 'wb') as f:\n#             pickle.dump(models_dict, f)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'models',\n#             self.config.models_path,\n#             'model'\n#         )\n        \n#         # Save configuration\n#         self.config.save()\n        \n#         # Create and save meta-features\n#         meta_features = self.create_meta_features(\n#             ensemble_result['oof_predictions'],\n#             train_df['label'].values,\n#             strategy_results\n#         )\n#         meta_features['row_id'] = train_df['original_index'].values\n        \n#         cols = ['row_id'] + [col for col in meta_features.columns if col != 'row_id']\n#         meta_features = meta_features[cols]\n        \n#         meta_features.to_csv(self.config.meta_features_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'meta_features',\n#             self.config.meta_features_path,\n#             'features',\n#             metadata={'n_features': len(meta_features.columns) - 1}\n#         )\n        \n#         # Save model performance summary\n#         performance_data = {\n#             'ensemble_score': float(ensemble_result['ensemble_score']),\n#             'strategy_scores': {k: float(v) for k, v in ensemble_result['strategy_scores'].items()},\n#             'ensemble_weights': ensemble_result['weights'],\n#             'best_strategy': max(ensemble_result['strategy_scores'].items(), key=lambda x: x[1])[0],\n#             'strategy_statistics': {}\n#         }\n        \n#         for strategy, result in strategy_results.items():\n#             if result['fold_scores']:\n#                 performance_data['strategy_statistics'][strategy] = {\n#                     'mean_score': float(np.mean(result['fold_scores'])),\n#                     'std_score': float(np.std(result['fold_scores'])),\n#                     'min_score': float(np.min(result['fold_scores'])),\n#                     'max_score': float(np.max(result['fold_scores']))\n#                 }\n        \n#         with open(self.config.model_performance_path, 'w') as f:\n#             json.dump(performance_data, f, indent=2)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'model_performance',\n#             self.config.model_performance_path,\n#             'analysis',\n#             metadata=performance_data\n#         )\n        \n#         print(f\"\\nAll outputs saved for {self.config.model_name}\")\n#         print(\"\\nStrategy Performance Summary:\")\n#         print(comparison_df.to_string(index=False))\n    \n#     def create_meta_features(self, predictions: np.ndarray, y_true: np.ndarray,\n#                            strategy_results: Dict[str, Dict[str, Any]]) -> pd.DataFrame:\n#         \"\"\"Create meta-features for meta-learning\"\"\"\n#         meta_features = {}\n        \n#         # Basic prediction features\n#         meta_features['prediction'] = predictions\n#         meta_features['error'] = predictions - y_true\n#         meta_features['abs_error'] = np.abs(predictions - y_true)\n#         meta_features['squared_error'] = (predictions - y_true) ** 2\n        \n#         # Strategy agreement features\n#         strategy_predictions = []\n#         for strategy, result in strategy_results.items():\n#             preds = result['oof_predictions']\n#             # Only include non-zero predictions\n#             mask = preds != 0\n#             if np.sum(mask) > 100:  # Minimum samples\n#                 strategy_predictions.append(preds)\n        \n#         if len(strategy_predictions) > 1:\n#             # Stack predictions\n#             pred_array = np.column_stack(strategy_predictions)\n            \n#             # Calculate statistics where all strategies have predictions\n#             valid_mask = np.all(pred_array != 0, axis=1)\n            \n#             if np.sum(valid_mask) > 0:\n#                 valid_preds = pred_array[valid_mask]\n                \n#                 # Initialize arrays\n#                 strategy_std = np.zeros(len(predictions))\n#                 strategy_range = np.zeros(len(predictions))\n#                 strategy_mean = np.zeros(len(predictions))\n                \n#                 # Fill in valid values\n#                 strategy_std[valid_mask] = np.std(valid_preds, axis=1)\n#                 strategy_range[valid_mask] = np.ptp(valid_preds, axis=1)\n#                 strategy_mean[valid_mask] = np.mean(valid_preds, axis=1)\n                \n#                 meta_features['strategy_std'] = strategy_std\n#                 meta_features['strategy_range'] = strategy_range\n#                 meta_features['strategy_mean'] = strategy_mean\n                \n#                 # Agreement score (inverse of std)\n#                 meta_features['strategy_agreement'] = 1 / (strategy_std + 1e-6)\n        \n#         # Best strategy indicator\n#         best_strategy = max(strategy_results.items(), key=lambda x: x[1]['overall_score'])[0]\n#         for strategy in strategy_results.keys():\n#             meta_features[f'is_best_{strategy}'] = (strategy == best_strategy).astype(int)\n        \n#         # Prediction percentiles\n#         valid_predictions = predictions[predictions != 0]\n#         if len(valid_predictions) > 0:\n#             percentiles = [10, 25, 50, 75, 90]\n#             for p in percentiles:\n#                 threshold = np.percentile(valid_predictions, p)\n#                 meta_features[f'above_p{p}'] = (predictions > threshold).astype(int)\n        \n#         return pd.DataFrame(meta_features)\n\n# # =============================================================================\n# # MAIN EXECUTION\n# # =============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\\n📊 Running Cross-Validation Strategy Ensemble Pipeline\")\n#     print(\"-\"*80)\n    \n#     # Clean memory\n#     aggressive_memory_cleanup()\n    \n#     # Create configuration\n#     config = CrossValidationStrategyConfiguration(\n#         n_splits=5,\n#         validation_strategies=[\n#             \"kfold_no_shuffle\",\n#             \"kfold_shuffle\",\n#             \"walk_forward_expanding\",\n#             \"walk_forward_sliding\"\n#         ],\n#         use_best_strategy_only=False,  # Use ensemble of strategies\n#         ensemble_weights_method=\"performance\",  # Weight by performance\n#         min_strategy_score=0.0  # Include all strategies\n#     )\n    \n#     # Create and run pipeline\n#     pipeline = CrossValidationStrategyPipeline(config)\n#     final_score = pipeline.run()","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # WIDE FEATURE ENSEMBLE PIPELINE - FULLY ENCAPSULATED IMPLEMENTATION\n# # !/usr/bin/env python\n# # -*- coding: utf-8 -*-\n# \"\"\"\n# DRW Crypto Market Prediction - Wide Feature Ensemble Pipeline\n# Utilizes many more features through intelligent subsampling and memory-efficient techniques\n# \"\"\"\n\n# import subprocess\n# import sys\n# import os\n# import gc\n# import warnings\n# import json\n# import pickle\n# import pandas as pd\n# import numpy as np\n# from scipy.stats import pearsonr\n# from scipy import sparse\n# from typing import List, Dict, Tuple, Optional, Any, Set\n# from pathlib import Path\n# from dataclasses import dataclass, field\n# from datetime import datetime\n# from sklearn.model_selection import train_test_split\n# from sklearn.feature_selection import (\n#     SelectKBest, f_regression, mutual_info_regression,\n#     chi2, SelectPercentile, VarianceThreshold\n# )\n# from sklearn.ensemble import RandomForestRegressor\n# from sklearn.preprocessing import StandardScaler\n# from sklearn.random_projection import SparseRandomProjection\n# from sklearn.decomposition import IncrementalPCA\n# import joblib\n\n# warnings.filterwarnings('ignore')\n\n# # Check that setup cell was run\n# try:\n#     global_config\n#     print(\"✓ Global configuration found\")\n# except NameError:\n#     raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# # Install required packages\n# print(\"Installing packages for Wide Feature Ensemble pipeline...\")\n# packages_to_install = [\n#     'xgboost==2.0.3',\n#     'lightgbm==4.1.0',\n#     'scikit-learn>=1.0.0',\n#     'joblib'\n# ]\n\n# for package in packages_to_install:\n#     try:\n#         subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n#     except:\n#         print(f\"Warning: Could not install {package}\")\n\n# # Import after installation\n# from xgboost import XGBRegressor\n# import lightgbm as lgb\n\n# # =============================================================================\n# # CONFIGURATION\n# # =============================================================================\n\n# @dataclass\n# class WideFeatureEnsembleConfiguration:\n#     \"\"\"Configuration for Wide Feature Ensemble Pipeline\"\"\"\n    \n#     # Model identification\n#     model_name: str = \"wide_feature_ensemble\"\n#     model_directory: str = \"\"\n    \n#     # Feature configuration\n#     n_feature_subsets: int = 10  # Number of different feature subsets to create\n#     features_per_subset: int = 150  # Number of features in each subset\n#     total_features_pool: int = 600  # Total unique features to consider\n#     min_feature_variance: float = 0.01  # Minimum variance threshold\n#     correlation_threshold: float = 0.95  # Remove highly correlated features\n    \n#     # Feature selection methods\n#     feature_selection_methods: List[str] = field(default_factory=list)\n    \n#     # Memory management\n#     use_sparse: bool = True  # Use sparse matrices where possible\n#     chunk_size: int = 50000  # Process data in chunks\n#     sample_size_for_selection: int = 100000  # Sample size for feature selection\n#     dtype_optimization: bool = True  # Optimize data types\n    \n#     # Model parameters\n#     n_models_per_subset: int = 2  # Number of models to train per feature subset\n#     model_types: List[str] = field(default_factory=list)\n    \n#     # XGBoost parameters (lighter for memory)\n#     xgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n#     # LightGBM parameters (lighter for memory)\n#     lgb_params: Dict[str, Any] = field(default_factory=dict)\n    \n#     # Training parameters\n#     validation_size: float = 0.2\n#     random_state: int = 42\n#     early_stopping_rounds: int = 50\n    \n#     # Ensemble parameters\n#     ensemble_method: str = \"weighted_average\"  # \"weighted_average\", \"stacking\", \"blending\"\n    \n#     def __post_init__(self):\n#         \"\"\"Initialize configuration and register model\"\"\"\n#         if not self.model_directory:\n#             self.model_directory = os.path.join(global_config.base_dir, \"wide_feature_ensemble\")\n        \n#         # Register model with global configuration\n#         global_config.register_model(self.model_name, self.model_directory)\n        \n#         # Default feature selection methods\n#         if not self.feature_selection_methods:\n#             self.feature_selection_methods = [\n#                 \"variance\",\n#                 \"f_statistic\",\n#                 \"mutual_info\",\n#                 \"random_forest\",\n#                 \"correlation\",\n#                 \"l1_regularization\"\n#             ]\n        \n#         # Default model types\n#         if not self.model_types:\n#             self.model_types = [\"xgboost\", \"lightgbm\"]\n        \n#         # Lightweight XGBoost parameters\n#         if not self.xgb_params:\n#             self.xgb_params = {\n#                 'n_estimators': 100,\n#                 'max_depth': 6,\n#                 'learning_rate': 0.1,\n#                 'subsample': 0.8,\n#                 'colsample_bytree': 0.8,\n#                 'colsample_bylevel': 0.8,\n#                 'colsample_bynode': 0.8,\n#                 'reg_alpha': 0.1,\n#                 'reg_lambda': 1.0,\n#                 'tree_method': 'hist',\n#                 'random_state': self.random_state,\n#                 'n_jobs': -1\n#             }\n        \n#         # Lightweight LightGBM parameters\n#         if not self.lgb_params:\n#             self.lgb_params = {\n#                 'objective': 'regression',\n#                 'metric': 'rmse',\n#                 'num_leaves': 31,\n#                 'learning_rate': 0.1,\n#                 'feature_fraction': 0.8,\n#                 'bagging_fraction': 0.8,\n#                 'bagging_freq': 5,\n#                 'min_data_in_leaf': 20,\n#                 'random_state': self.random_state,\n#                 'n_jobs': -1,\n#                 'force_col_wise': True,\n#                 'min_data_per_group': 100,\n#                 'max_cat_threshold': 32,\n#                 'cat_l2': 10,\n#                 'cat_smooth': 10,\n#                 'max_cat_to_onehot': 4\n#             }\n        \n#         # Output paths\n#         self.submission_path = os.path.join(self.model_directory, \"submission.csv\")\n#         self.feature_subsets_path = os.path.join(self.model_directory, \"feature_subsets.json\")\n#         self.feature_importance_path = os.path.join(self.model_directory, \"feature_importance.csv\")\n#         self.models_path = os.path.join(self.model_directory, \"models\")\n#         self.config_path = os.path.join(self.model_directory, \"config.json\")\n        \n#         # Meta-learning paths\n#         self.oof_predictions_path = os.path.join(self.model_directory, \"oof_predictions.csv\")\n#         self.meta_features_path = os.path.join(self.model_directory, \"meta_features.csv\")\n#         self.model_performance_path = os.path.join(self.model_directory, \"model_performance.json\")\n#         self.feature_usage_path = os.path.join(self.model_directory, \"feature_usage_stats.csv\")\n        \n#         # Ensure directories exist\n#         Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n#         Path(self.models_path).mkdir(parents=True, exist_ok=True)\n    \n#     def save(self):\n#         \"\"\"Save configuration to JSON file\"\"\"\n#         config_dict = {\n#             'model_name': self.model_name,\n#             'n_feature_subsets': self.n_feature_subsets,\n#             'features_per_subset': self.features_per_subset,\n#             'total_features_pool': self.total_features_pool,\n#             'feature_selection_methods': self.feature_selection_methods,\n#             'model_types': self.model_types,\n#             'use_sparse': self.use_sparse,\n#             'chunk_size': self.chunk_size,\n#             'xgb_params': self.xgb_params,\n#             'lgb_params': self.lgb_params,\n#             'random_state': self.random_state\n#         }\n#         with open(self.config_path, 'w') as f:\n#             json.dump(config_dict, f, indent=2)\n\n# # =============================================================================\n# # MEMORY-EFFICIENT DATA PROCESSING\n# # =============================================================================\n\n# class MemoryEfficientDataProcessor:\n#     \"\"\"Process data with minimal memory footprint\"\"\"\n    \n#     def __init__(self, config: WideFeatureEnsembleConfiguration):\n#         self.config = config\n    \n#     def optimize_dtypes(self, df: pd.DataFrame) -> pd.DataFrame:\n#         \"\"\"Optimize data types to reduce memory usage\"\"\"\n#         for col in df.columns:\n#             col_type = df[col].dtype\n            \n#             if col_type != object:\n#                 c_min = df[col].min()\n#                 c_max = df[col].max()\n                \n#                 if str(col_type)[:3] == 'int':\n#                     if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n#                         df[col] = df[col].astype(np.int8)\n#                     elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n#                         df[col] = df[col].astype(np.int16)\n#                     elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n#                         df[col] = df[col].astype(np.int32)\n#                 else:\n#                     if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n#                         df[col] = df[col].astype(np.float16)\n#                     elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n#                         df[col] = df[col].astype(np.float32)\n        \n#         return df\n    \n#     def create_sparse_features(self, X: np.ndarray, threshold: float = 0.1) -> sparse.csr_matrix:\n#         \"\"\"Convert to sparse matrix if beneficial\"\"\"\n#         sparsity = np.mean(X == 0)\n#         if sparsity > threshold:\n#             return sparse.csr_matrix(X)\n#         return X\n\n# # =============================================================================\n# # DIVERSE FEATURE SELECTOR\n# # =============================================================================\n\n# class DiverseFeatureSelector:\n#     \"\"\"Select diverse sets of features using multiple methods\"\"\"\n    \n#     def __init__(self, config: WideFeatureEnsembleConfiguration):\n#         self.config = config\n#         self.feature_scores = {}\n#         self.selected_features = {}\n    \n#     def select_features(self, X: pd.DataFrame, y: np.ndarray) -> Dict[int, Dict[str, Any]]:\n#         \"\"\"Select multiple diverse feature subsets\"\"\"\n#         print(f\"\\nSelecting {self.config.n_feature_subsets} diverse feature subsets...\")\n        \n#         # Get all numeric features\n#         numeric_features = X.select_dtypes(include=[np.number]).columns.tolist()\n#         numeric_features = [f for f in numeric_features if f not in ['label', 'timestamp', 'original_index']]\n        \n#         print(f\"Total numeric features available: {len(numeric_features)}\")\n        \n#         # Sample data for feature selection if needed\n#         if len(X) > self.config.sample_size_for_selection:\n#             sample_idx = np.random.choice(len(X), self.config.sample_size_for_selection, replace=False)\n#             X_sample = X.iloc[sample_idx]\n#             y_sample = y[sample_idx]\n#         else:\n#             X_sample = X\n#             y_sample = y\n        \n#         # 1. Remove low variance features\n#         print(\"  Removing low variance features...\")\n#         variance_selector = VarianceThreshold(threshold=self.config.min_feature_variance)\n#         variance_selector.fit(X_sample[numeric_features])\n#         high_variance_features = [f for f, keep in zip(numeric_features, variance_selector.get_support()) if keep]\n#         print(f\"    Features after variance filter: {len(high_variance_features)}\")\n        \n#         # 2. Remove highly correlated features\n#         print(\"  Removing highly correlated features...\")\n#         correlation_matrix = X_sample[high_variance_features].corr().abs()\n#         upper_triangle = correlation_matrix.where(\n#             np.triu(np.ones(correlation_matrix.shape), k=1).astype(bool)\n#         )\n        \n#         to_drop = [column for column in upper_triangle.columns \n#                    if any(upper_triangle[column] > self.config.correlation_threshold)]\n        \n#         uncorrelated_features = [f for f in high_variance_features if f not in to_drop]\n#         print(f\"    Features after correlation filter: {len(uncorrelated_features)}\")\n        \n#         # Ensure we have enough features\n#         feature_pool = uncorrelated_features[:self.config.total_features_pool]\n        \n#         if len(feature_pool) < self.config.features_per_subset:\n#             print(f\"  Warning: Only {len(feature_pool)} features available\")\n#             feature_pool = numeric_features[:self.config.total_features_pool]\n        \n#         # 3. Apply different selection methods\n#         feature_subsets = {}\n#         used_features = set()\n        \n#         for subset_idx in range(self.config.n_feature_subsets):\n#             print(f\"\\n  Creating subset {subset_idx + 1}/{self.config.n_feature_subsets}\")\n            \n#             # Choose selection method\n#             method_idx = subset_idx % len(self.config.feature_selection_methods)\n#             method = self.config.feature_selection_methods[method_idx]\n            \n#             # Get candidate features (prefer unused features)\n#             unused_features = [f for f in feature_pool if f not in used_features]\n#             if len(unused_features) < self.config.features_per_subset:\n#                 candidate_features = feature_pool\n#             else:\n#                 candidate_features = unused_features\n            \n#             # Apply selection method\n#             selected = self._apply_selection_method(\n#                 X_sample[candidate_features], \n#                 y_sample, \n#                 method, \n#                 self.config.features_per_subset\n#             )\n            \n#             # Update tracking\n#             selected_feature_names = [candidate_features[i] for i in selected]\n#             used_features.update(selected_feature_names)\n            \n#             feature_subsets[subset_idx] = {\n#                 'features': selected_feature_names,\n#                 'method': method,\n#                 'n_features': len(selected_feature_names)\n#             }\n            \n#             print(f\"    Method: {method}, Selected: {len(selected_feature_names)} features\")\n        \n#         return feature_subsets\n    \n#     def _apply_selection_method(self, X: pd.DataFrame, y: np.ndarray, \n#                                method: str, n_features: int) -> List[int]:\n#         \"\"\"Apply a specific feature selection method\"\"\"\n#         n_features = min(n_features, X.shape[1])\n        \n#         if method == \"variance\":\n#             # Select features with highest variance\n#             variances = X.var()\n#             return variances.nlargest(n_features).index.tolist()\n        \n#         elif method == \"f_statistic\":\n#             # F-statistic selection\n#             selector = SelectKBest(score_func=f_regression, k=n_features)\n#             selector.fit(X, y)\n#             return np.where(selector.get_support())[0].tolist()\n        \n#         elif method == \"mutual_info\":\n#             # Mutual information selection\n#             selector = SelectKBest(score_func=mutual_info_regression, k=n_features)\n#             selector.fit(X, y)\n#             return np.where(selector.get_support())[0].tolist()\n        \n#         elif method == \"random_forest\":\n#             # Random forest importance\n#             rf = RandomForestRegressor(\n#                 n_estimators=50, \n#                 max_depth=5, \n#                 random_state=self.config.random_state,\n#                 n_jobs=-1\n#             )\n#             rf.fit(X, y)\n#             importance_idx = np.argsort(rf.feature_importances_)[-n_features:]\n#             return importance_idx.tolist()\n        \n#         elif method == \"correlation\":\n#             # Correlation with target\n#             correlations = X.corrwith(pd.Series(y)).abs()\n#             return correlations.nlargest(n_features).index.tolist()\n        \n#         elif method == \"l1_regularization\":\n#             # L1-based selection using LightGBM\n#             lgb_data = lgb.Dataset(X, label=y)\n#             params = {\n#                 'objective': 'regression',\n#                 'metric': 'rmse',\n#                 'num_leaves': 31,\n#                 'learning_rate': 0.1,\n#                 'feature_fraction': 0.8,\n#                 'bagging_fraction': 0.8,\n#                 'lambda_l1': 1.0,\n#                 'verbosity': -1\n#             }\n            \n#             model = lgb.train(params, lgb_data, num_boost_round=100)\n#             importance_idx = np.argsort(model.feature_importance())[-n_features:]\n#             return importance_idx.tolist()\n        \n#         else:\n#             # Random selection as fallback\n#             return np.random.choice(X.shape[1], size=n_features, replace=False).tolist()\n\n# # =============================================================================\n# # SUBSET MODEL TRAINER\n# # =============================================================================\n\n# class SubsetModelTrainer:\n#     \"\"\"Train models on different feature subsets\"\"\"\n    \n#     def __init__(self, config: WideFeatureEnsembleConfiguration):\n#         self.config = config\n#         self.models = {}\n#         self.subset_scores = {}\n    \n#     def train_subset_models(self, X_train: pd.DataFrame, y_train: np.ndarray,\n#                            X_test: pd.DataFrame, feature_subsets: Dict[int, Dict[str, Any]]) -> Dict[str, Any]:\n#         \"\"\"Train models on each feature subset\"\"\"\n#         print(\"\\nTraining models on feature subsets...\")\n        \n#         # Split for validation\n#         X_tr, X_val, y_tr, y_val = train_test_split(\n#             X_train, y_train, \n#             test_size=self.config.validation_size,\n#             random_state=self.config.random_state\n#         )\n        \n#         all_predictions = {\n#             'oof': {},\n#             'test': {}\n#         }\n        \n#         subset_results = {}\n        \n#         for subset_idx, subset_info in feature_subsets.items():\n#             print(f\"\\n  Subset {subset_idx + 1}: {len(subset_info['features'])} features ({subset_info['method']})\")\n            \n#             features = subset_info['features']\n            \n#             # Prepare data for this subset\n#             X_tr_subset = X_tr[features].values\n#             X_val_subset = X_val[features].values\n#             X_test_subset = X_test[features].values\n            \n#             subset_predictions = {\n#                 'oof': np.zeros(len(X_train)),\n#                 'test': np.zeros(len(X_test))\n#             }\n            \n#             model_scores = []\n            \n#             # Train different model types on this subset\n#             for model_type in self.config.model_types[:self.config.n_models_per_subset]:\n#                 print(f\"    Training {model_type}...\")\n                \n#                 try:\n#                     if model_type == \"xgboost\":\n#                         model = XGBRegressor(**self.config.xgb_params)\n#                         model.fit(\n#                             X_tr_subset, y_tr,\n#                             eval_set=[(X_val_subset, y_val)],\n#                             early_stopping_rounds=self.config.early_stopping_rounds,\n#                             verbose=False\n#                         )\n                        \n#                     elif model_type == \"lightgbm\":\n#                         train_data = lgb.Dataset(X_tr_subset, label=y_tr)\n#                         val_data = lgb.Dataset(X_val_subset, label=y_val, reference=train_data)\n                        \n#                         model = lgb.train(\n#                             self.config.lgb_params,\n#                             train_data,\n#                             valid_sets=[val_data],\n#                             num_boost_round=300,\n#                             callbacks=[\n#                                 lgb.early_stopping(self.config.early_stopping_rounds),\n#                                 lgb.log_evaluation(0)\n#                             ]\n#                         )\n                    \n#                     # Make predictions\n#                     if isinstance(model, lgb.Booster):\n#                         val_pred = model.predict(X_val_subset, num_iteration=model.best_iteration)\n#                         test_pred = model.predict(X_test_subset, num_iteration=model.best_iteration)\n                        \n#                         # Full OOF predictions\n#                         oof_pred = np.zeros(len(X_train))\n#                         oof_pred[X_val.index] = val_pred\n#                     else:\n#                         val_pred = model.predict(X_val_subset)\n#                         test_pred = model.predict(X_test_subset)\n                        \n#                         # Full OOF predictions\n#                         oof_pred = np.zeros(len(X_train))\n#                         oof_pred[X_val.index] = val_pred\n                    \n#                     # Calculate score\n#                     score = pearsonr(y_val, val_pred)[0]\n#                     model_scores.append(score)\n                    \n#                     print(f\"      Score: {score:.4f}\")\n                    \n#                     # Store predictions\n#                     model_key = f\"subset_{subset_idx}_{model_type}\"\n#                     all_predictions['oof'][model_key] = oof_pred\n#                     all_predictions['test'][model_key] = test_pred\n                    \n#                     # Save model\n#                     model_path = os.path.join(self.config.models_path, f\"{model_key}.pkl\")\n#                     joblib.dump(model, model_path)\n                    \n#                     # Update subset predictions (average)\n#                     subset_predictions['oof'] += oof_pred / self.config.n_models_per_subset\n#                     subset_predictions['test'] += test_pred / self.config.n_models_per_subset\n                    \n#                 except Exception as e:\n#                     print(f\"      Error: {e}\")\n#                     continue\n            \n#             # Store subset results\n#             if model_scores:\n#                 subset_results[subset_idx] = {\n#                     'features': features,\n#                     'method': subset_info['method'],\n#                     'scores': model_scores,\n#                     'mean_score': np.mean(model_scores),\n#                     'predictions': subset_predictions\n#                 }\n        \n#         return {\n#             'all_predictions': all_predictions,\n#             'subset_results': subset_results,\n#             'val_indices': X_val.index.tolist()\n#         }\n\n# # =============================================================================\n# # ENSEMBLE BUILDER\n# # =============================================================================\n\n# class WideFeatureEnsembleBuilder:\n#     \"\"\"Build ensemble from subset models\"\"\"\n    \n#     def __init__(self, config: WideFeatureEnsembleConfiguration):\n#         self.config = config\n    \n#     def create_ensemble(self, training_results: Dict[str, Any], \n#                        y_train: np.ndarray) -> Dict[str, Any]:\n#         \"\"\"Create ensemble from all model predictions\"\"\"\n#         print(\"\\nCreating final ensemble...\")\n        \n#         all_predictions = training_results['all_predictions']\n#         subset_results = training_results['subset_results']\n#         val_indices = training_results['val_indices']\n        \n#         # Calculate model weights based on validation performance\n#         model_scores = {}\n#         for model_key, oof_pred in all_predictions['oof'].items():\n#             # Only evaluate on validation indices\n#             val_pred = oof_pred[val_indices]\n#             val_true = y_train[val_indices]\n            \n#             if np.any(val_pred != 0):\n#                 score = pearsonr(val_true, val_pred)[0]\n#                 model_scores[model_key] = max(0, score)  # Ensure non-negative\n#             else:\n#                 model_scores[model_key] = 0\n        \n#         # Create weighted ensemble\n#         if self.config.ensemble_method == \"weighted_average\":\n#             # Normalize scores to weights\n#             total_score = sum(model_scores.values())\n#             if total_score > 0:\n#                 weights = {k: v/total_score for k, v in model_scores.items()}\n#             else:\n#                 weights = {k: 1/len(model_scores) for k in model_scores}\n            \n#             # Create ensemble predictions\n#             ensemble_oof = np.zeros(len(all_predictions['oof'][next(iter(all_predictions['oof']))]))\n#             ensemble_test = np.zeros(len(all_predictions['test'][next(iter(all_predictions['test']))]))\n            \n#             for model_key, weight in weights.items():\n#                 ensemble_oof += weight * all_predictions['oof'][model_key]\n#                 ensemble_test += weight * all_predictions['test'][model_key]\n        \n#         else:\n#             # Simple average fallback\n#             ensemble_oof = np.mean(list(all_predictions['oof'].values()), axis=0)\n#             ensemble_test = np.mean(list(all_predictions['test'].values()), axis=0)\n#             weights = {k: 1/len(model_scores) for k in model_scores}\n        \n#         # Calculate ensemble score on validation set\n#         val_ensemble = ensemble_oof[val_indices]\n#         val_true = y_train[val_indices]\n#         ensemble_score = pearsonr(val_true, val_ensemble)[0]\n        \n#         print(f\"\\nEnsemble validation score: {ensemble_score:.4f}\")\n        \n#         # Print top models\n#         print(\"\\nTop performing models:\")\n#         for model_key, score in sorted(model_scores.items(), key=lambda x: x[1], reverse=True)[:5]:\n#             print(f\"  {model_key}: {score:.4f} (weight: {weights[model_key]:.3f})\")\n        \n#         return {\n#             'ensemble_oof': ensemble_oof,\n#             'ensemble_test': ensemble_test,\n#             'ensemble_score': ensemble_score,\n#             'model_scores': model_scores,\n#             'weights': weights,\n#             'subset_results': subset_results\n#         }\n\n# # =============================================================================\n# # MAIN PIPELINE\n# # =============================================================================\n\n# class WideFeatureEnsemblePipeline:\n#     \"\"\"Main pipeline for wide feature ensemble\"\"\"\n    \n#     def __init__(self, config: Optional[WideFeatureEnsembleConfiguration] = None):\n#         self.config = config or WideFeatureEnsembleConfiguration()\n#         self.data_processor = MemoryEfficientDataProcessor(self.config)\n#         self.feature_selector = DiverseFeatureSelector(self.config)\n#         self.model_trainer = SubsetModelTrainer(self.config)\n#         self.ensemble_builder = WideFeatureEnsembleBuilder(self.config)\n    \n#     def load_and_prepare_data(self) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:\n#         \"\"\"Load and prepare data\"\"\"\n#         print(\"Loading data...\")\n        \n#         # Load data\n#         train_df = pd.read_parquet(global_config.train_path)\n#         test_df = pd.read_parquet(global_config.test_path)\n#         sample_submission = pd.read_csv(global_config.sample_sub_path)\n        \n#         # Add indices\n#         train_df['original_index'] = range(len(train_df))\n#         test_df['original_index'] = range(len(test_df))\n        \n#         # Optimize memory\n#         if self.config.dtype_optimization:\n#             print(\"Optimizing data types...\")\n#             train_df = self.data_processor.optimize_dtypes(train_df)\n#             test_df = self.data_processor.optimize_dtypes(test_df)\n        \n#         print(f\"Data loaded - Train: {train_df.shape}, Test: {test_df.shape}\")\n        \n#         # Remove constant features\n#         print(\"Removing constant features...\")\n#         numeric_cols = train_df.select_dtypes(include=[np.number]).columns\n#         constant_features = []\n        \n#         for col in numeric_cols:\n#             if train_df[col].nunique() <= 1:\n#                 constant_features.append(col)\n        \n#         if constant_features:\n#             print(f\"  Removing {len(constant_features)} constant features\")\n#             train_df = train_df.drop(constant_features, axis=1)\n#             test_df = test_df.drop([f for f in constant_features if f in test_df.columns], axis=1)\n        \n#         gc.collect()\n        \n#         return train_df, test_df, sample_submission\n    \n#     def run(self) -> float:\n#         \"\"\"Execute the complete pipeline\"\"\"\n#         print(\"\\nStarting Wide Feature Ensemble Pipeline\")\n#         print(\"=\"*80)\n        \n#         # Update status\n#         global_config.update_model_status(self.config.model_name, 'running')\n        \n#         try:\n#             # Load and prepare data\n#             train_df, test_df, sample_submission = self.load_and_prepare_data()\n            \n#             # Extract labels and indices\n#             y_train = train_df['label'].values\n#             train_indices = train_df['original_index'].values\n            \n#             # Select diverse feature subsets\n#             feature_subsets = self.feature_selector.select_features(train_df, y_train)\n            \n#             # Save feature subsets\n#             feature_subsets_serializable = {\n#                 str(k): {\n#                     'features': v['features'],\n#                     'method': v['method'],\n#                     'n_features': v['n_features']\n#                 }\n#                 for k, v in feature_subsets.items()\n#             }\n            \n#             with open(self.config.feature_subsets_path, 'w') as f:\n#                 json.dump(feature_subsets_serializable, f, indent=2)\n            \n#             # Train models on subsets\n#             training_results = self.model_trainer.train_subset_models(\n#                 train_df, y_train, test_df, feature_subsets\n#             )\n            \n#             # Create ensemble\n#             ensemble_results = self.ensemble_builder.create_ensemble(\n#                 training_results, y_train\n#             )\n            \n#             # Save outputs\n#             self.save_outputs(\n#                 ensemble_results=ensemble_results,\n#                 feature_subsets=feature_subsets,\n#                 train_indices=train_indices,\n#                 y_train=y_train,\n#                 sample_submission=sample_submission\n#             )\n            \n#             # Update status\n#             final_score = ensemble_results['ensemble_score']\n#             global_config.update_model_status(\n#                 self.config.model_name,\n#                 'completed',\n#                 score=final_score\n#             )\n            \n#             print(f\"\\n✅ Wide Feature Ensemble pipeline completed successfully\")\n#             print(f\"  Final Score: {final_score:.6f}\")\n#             print(f\"  Total features used: {len(set().union(*[s['features'] for s in feature_subsets.values()]))}\")\n            \n#             return final_score\n            \n#         except Exception as e:\n#             error_msg = str(e)\n#             global_config.update_model_status(self.config.model_name, 'failed', error_message=error_msg)\n#             print(f\"\\n❌ Wide Feature Ensemble pipeline failed: {error_msg}\")\n#             raise\n            \n#         finally:\n#             aggressive_memory_cleanup()\n#             print(\"\\n\" + \"=\"*80)\n#             print(\"Current Execution Status:\")\n#             print(global_config.get_execution_summary())\n#             print(\"=\"*80)\n    \n#     def save_outputs(self, ensemble_results, feature_subsets, train_indices, y_train, sample_submission):\n#         \"\"\"Save all outputs\"\"\"\n        \n#         # Save submission\n#         submission = sample_submission.copy()\n#         submission['prediction'] = ensemble_results['ensemble_test']\n#         submission.to_csv(self.config.submission_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'submission',\n#             self.config.submission_path,\n#             'submission',\n#             metadata={\n#                 'score': float(ensemble_results['ensemble_score']),\n#                 'n_models': len(ensemble_results['model_scores']),\n#                 'n_subsets': len(feature_subsets)\n#             }\n#         )\n        \n#         # Save OOF predictions\n#         oof_df = pd.DataFrame({\n#             'row_id': train_indices,\n#             'true_label': y_train,\n#             'prediction': ensemble_results['ensemble_oof']\n#         })\n#         oof_df.to_csv(self.config.oof_predictions_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'oof_predictions',\n#             self.config.oof_predictions_path,\n#             'oof_predictions'\n#         )\n        \n#         # Save feature importance/usage statistics\n#         feature_usage = {}\n#         for subset_idx, subset_info in feature_subsets.items():\n#             for feature in subset_info['features']:\n#                 if feature not in feature_usage:\n#                     feature_usage[feature] = {\n#                         'count': 0,\n#                         'subsets': [],\n#                         'methods': []\n#                     }\n#                 feature_usage[feature]['count'] += 1\n#                 feature_usage[feature]['subsets'].append(subset_idx)\n#                 feature_usage[feature]['methods'].append(subset_info['method'])\n        \n#         # Create feature usage dataframe\n#         usage_data = []\n#         for feature, info in feature_usage.items():\n#             usage_data.append({\n#                 'feature': feature,\n#                 'usage_count': info['count'],\n#                 'n_subsets': len(set(info['subsets'])),\n#                 'methods': ','.join(set(info['methods']))\n#             })\n        \n#         usage_df = pd.DataFrame(usage_data).sort_values('usage_count', ascending=False)\n#         usage_df.to_csv(self.config.feature_usage_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'feature_usage',\n#             self.config.feature_usage_path,\n#             'analysis',\n#             metadata={\n#                 'total_unique_features': len(feature_usage),\n#                 'most_used_features': usage_df.head(10)['feature'].tolist()\n#             }\n#         )\n        \n#         # Save model performance\n#         performance_data = {\n#             'ensemble_score': float(ensemble_results['ensemble_score']),\n#             'model_scores': {k: float(v) for k, v in ensemble_results['model_scores'].items()},\n#             'weights': {k: float(v) for k, v in ensemble_results['weights'].items()},\n#             'subset_performance': {}\n#         }\n        \n#         for subset_idx, subset_result in ensemble_results['subset_results'].items():\n#             performance_data['subset_performance'][str(subset_idx)] = {\n#                 'method': subset_result['method'],\n#                 'n_features': len(subset_result['features']),\n#                 'mean_score': float(subset_result['mean_score']),\n#                 'individual_scores': [float(s) for s in subset_result['scores']]\n#             }\n        \n#         with open(self.config.model_performance_path, 'w') as f:\n#             json.dump(performance_data, f, indent=2)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'model_performance',\n#             self.config.model_performance_path,\n#             'analysis'\n#         )\n        \n#         # Save configuration\n#         self.config.save()\n        \n#         # Create meta features\n#         meta_features = self.create_meta_features(\n#             ensemble_results['ensemble_oof'],\n#             y_train,\n#             ensemble_results\n#         )\n#         meta_features['row_id'] = train_indices\n        \n#         cols = ['row_id'] + [col for col in meta_features.columns if col != 'row_id']\n#         meta_features = meta_features[cols]\n        \n#         meta_features.to_csv(self.config.meta_features_path, index=False)\n        \n#         global_config.register_model_output(\n#             self.config.model_name,\n#             'meta_features',\n#             self.config.meta_features_path,\n#             'features'\n#         )\n        \n#         print(f\"\\nAll outputs saved for {self.config.model_name}\")\n#         print(f\"\\nFeature usage summary:\")\n#         print(f\"  Total unique features used: {len(feature_usage)}\")\n#         print(f\"  Average features per subset: {np.mean([len(s['features']) for s in feature_subsets.values()]):.1f}\")\n#         print(f\"\\nTop 10 most used features:\")\n#         for _, row in usage_df.head(10).iterrows():\n#             print(f\"  {row['feature']}: used {row['usage_count']} times\")\n    \n#     def create_meta_features(self, predictions, y_true, ensemble_results):\n#         \"\"\"Create meta-features\"\"\"\n#         meta_features = {}\n        \n#         # Basic features\n#         meta_features['prediction'] = predictions\n#         meta_features['error'] = predictions - y_true\n#         meta_features['abs_error'] = np.abs(predictions - y_true)\n#         meta_features['squared_error'] = (predictions - y_true) ** 2\n        \n#         # Subset diversity features\n#         subset_predictions = []\n#         for subset_result in ensemble_results['subset_results'].values():\n#             if 'predictions' in subset_result:\n#                 subset_predictions.append(subset_result['predictions']['oof'])\n        \n#         if len(subset_predictions) > 1:\n#             subset_array = np.column_stack(subset_predictions)\n#             meta_features['subset_std'] = np.std(subset_array, axis=1)\n#             meta_features['subset_range'] = np.ptp(subset_array, axis=1)\n        \n#         return pd.DataFrame(meta_features)\n\n# # =============================================================================\n# # MAIN EXECUTION\n# # =============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\\n🌐 Running Wide Feature Ensemble Pipeline\")\n#     print(\"-\"*80)\n    \n#     # Clean memory\n#     aggressive_memory_cleanup()\n    \n#     # Create configuration\n#     config = WideFeatureEnsembleConfiguration(\n#         n_feature_subsets=10,  # Create 10 different feature subsets\n#         features_per_subset=150,  # 150 features per subset\n#         total_features_pool=600,  # Consider top 600 features\n#         feature_selection_methods=[\n#             \"variance\",\n#             \"f_statistic\", \n#             \"mutual_info\",\n#             \"random_forest\",\n#             \"correlation\",\n#             \"l1_regularization\"\n#         ],\n#         model_types=[\"xgboost\", \"lightgbm\"],\n#         n_models_per_subset=2,  # Train 2 models per subset\n#         use_sparse=True,\n#         chunk_size=50000\n#     )\n    \n#     # Create and run pipeline\n#     pipeline = WideFeatureEnsemblePipeline(config)\n#     final_score = pipeline.run()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stable Meta-Learning Ensemble with Non-Collinear Transformations\n# !/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nDRW Crypto Market Prediction - Stable Meta-Learning Ensemble Builder\nFocuses on differences, PCA, and non-collinear transformations to avoid multicollinearity\n\"\"\"\n\nimport subprocess\nimport sys\nimport os\nimport gc\nimport warnings\nimport json\nimport pickle\nimport pandas as pd\nimport numpy as np\nfrom scipy.stats import pearsonr, spearmanr, rankdata\nfrom scipy.special import logit, expit\nfrom typing import List, Dict, Tuple, Optional, Any, Union\nfrom pathlib import Path\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, QuantileTransformer\nfrom sklearn.linear_model import Ridge, ElasticNet, HuberRegressor\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\nfrom sklearn.decomposition import PCA, FastICA\nfrom sklearn.feature_selection import SelectKBest, f_regression\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nwarnings.filterwarnings('ignore')\n\n# Check that setup cell was run\ntry:\n    global_config\n    print(\"✓ Global configuration found\")\nexcept NameError:\n    raise RuntimeError(\"Please run the setup cell first! global_config is not defined.\")\n\n# Install required packages\nprint(\"Installing packages for Stable Meta-Learning Ensemble...\")\npackages_to_install = [\n    'lightgbm>=4.0.0',\n    'matplotlib',\n    'seaborn',\n    'joblib'\n]\n\nfor package in packages_to_install:\n    try:\n        subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package, \"--quiet\"])\n    except:\n        print(f\"Warning: Could not install {package}\")\n\nimport joblib\n\n# =============================================================================\n# Non-Collinear Feature Engineering\n# =============================================================================\n\nclass NonCollinearFeatureEngineer:\n    \"\"\"Create non-collinear features from model predictions\"\"\"\n    \n    def __init__(self, n_components: int = 10, random_state: int = 42):\n        self.n_components = n_components\n        self.random_state = random_state\n        self.pca = None\n        self.ica = None\n        self.scaler = StandardScaler()\n        self.quantile_transformer = QuantileTransformer(output_distribution='normal', random_state=random_state)\n        self.feature_names = []\n        \n    def create_difference_features(self, predictions: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:\n        \"\"\"Create pairwise difference features to capture relative performance\"\"\"\n        diff_features = {}\n        model_names = list(predictions.keys())\n        \n        # Pairwise differences\n        for i in range(len(model_names)):\n            for j in range(i + 1, len(model_names)):\n                name1, name2 = model_names[i], model_names[j]\n                \n                # Simple difference\n                diff_features[f'diff_{name1}_minus_{name2}'] = predictions[name1] - predictions[name2]\n                \n                # Relative difference (percentage)\n                mean_pred = (predictions[name1] + predictions[name2]) / 2\n                safe_mean = np.where(np.abs(mean_pred) < 1e-8, 1e-8, mean_pred)\n                diff_features[f'reldiff_{name1}_vs_{name2}'] = (predictions[name1] - predictions[name2]) / np.abs(safe_mean)\n                \n                # Log ratio (for positive predictions)\n                min_val = min(predictions[name1].min(), predictions[name2].min())\n                if min_val > 0:\n                    diff_features[f'logratio_{name1}_over_{name2}'] = np.log(predictions[name1] / predictions[name2])\n        \n        return diff_features\n    \n    def create_rank_deviation_features(self, predictions: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:\n        \"\"\"Create features based on rank deviations\"\"\"\n        rank_features = {}\n        \n        # Convert predictions to ranks\n        ranks = {}\n        for name, pred in predictions.items():\n            ranks[name] = rankdata(pred) / len(pred)\n        \n        # Mean rank across models\n        mean_rank = np.mean(list(ranks.values()), axis=0)\n        \n        # Rank deviations from consensus\n        for name, rank in ranks.items():\n            rank_features[f'{name}_rank_deviation'] = rank - mean_rank\n            rank_features[f'{name}_rank_abs_deviation'] = np.abs(rank - mean_rank)\n        \n        # Rank dispersion (how much models disagree on ranking)\n        rank_array = np.column_stack(list(ranks.values()))\n        rank_features['rank_dispersion'] = np.std(rank_array, axis=1)\n        rank_features['rank_iqr'] = np.percentile(rank_array, 75, axis=1) - np.percentile(rank_array, 25, axis=1)\n        \n        return rank_features\n    \n    def create_nonlinear_transforms(self, predictions: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:\n        \"\"\"Create non-linear transformations that are less correlated\"\"\"\n        nonlinear_features = {}\n        \n        # For each prediction, create unique transformations\n        for name, pred in predictions.items():\n            # Normalize to [0, 1] for stable transforms\n            pred_norm = (pred - pred.min()) / (pred.max() - pred.min() + 1e-8)\n            \n            # Logit transform (S-curve) - emphasizes extremes\n            pred_logit_safe = np.clip(pred_norm, 0.01, 0.99)\n            nonlinear_features[f'{name}_logit'] = logit(pred_logit_safe)\n            \n            # Sine transform - creates cyclical patterns\n            nonlinear_features[f'{name}_sin'] = np.sin(2 * np.pi * pred_norm)\n            \n            # Exponential decay from median\n            median_val = np.median(pred)\n            nonlinear_features[f'{name}_exp_decay'] = np.exp(-np.abs(pred - median_val) / np.std(pred))\n        \n        return nonlinear_features\n    \n    def create_local_statistics(self, predictions: Dict[str, np.ndarray], window_pct: float = 0.1) -> Dict[str, np.ndarray]:\n        \"\"\"Create local statistics to capture temporal patterns\"\"\"\n        local_features = {}\n        pred_array = np.column_stack(list(predictions.values()))\n        n_samples = len(pred_array)\n        window_size = max(10, int(n_samples * window_pct))\n        \n        # Rolling statistics with padding\n        for i, (name, pred) in enumerate(predictions.items()):\n            rolling_mean = np.convolve(pred, np.ones(window_size)/window_size, mode='same')\n            rolling_std = pd.Series(pred).rolling(window_size, center=True, min_periods=1).std().values\n            \n            # Deviation from local mean\n            local_features[f'{name}_local_deviation'] = pred - rolling_mean\n            \n            # Normalized local deviation\n            local_features[f'{name}_local_zscore'] = np.where(\n                rolling_std > 1e-8,\n                (pred - rolling_mean) / rolling_std,\n                0\n            )\n        \n        # Local model agreement\n        local_corr = []\n        for i in range(n_samples):\n            start = max(0, i - window_size // 2)\n            end = min(n_samples, i + window_size // 2)\n            if end - start > 2:\n                window_data = pred_array[start:end]\n                corr_matrix = np.corrcoef(window_data.T)\n                # Average off-diagonal correlation\n                mask = ~np.eye(corr_matrix.shape[0], dtype=bool)\n                avg_corr = np.mean(corr_matrix[mask])\n                local_corr.append(avg_corr)\n            else:\n                local_corr.append(0.5)\n        \n        local_features['local_model_agreement'] = np.array(local_corr)\n        \n        return local_features\n    \n    def apply_pca_transformation(self, features: np.ndarray, fit: bool = True) -> np.ndarray:\n        \"\"\"Apply PCA to decorrelate features\"\"\"\n        if fit:\n            # Fit PCA to capture most variance with fewer components\n            self.pca = PCA(n_components=min(self.n_components, features.shape[1]), \n                           random_state=self.random_state)\n            pca_features = self.pca.fit_transform(features)\n            \n            # Print explained variance\n            cumsum_var = np.cumsum(self.pca.explained_variance_ratio_)\n            print(f\"  PCA: {self.pca.n_components_} components explain {cumsum_var[-1]:.3f} of variance\")\n        else:\n            pca_features = self.pca.transform(features)\n        \n        return pca_features\n    \n    def apply_ica_transformation(self, features: np.ndarray, fit: bool = True) -> np.ndarray:\n        \"\"\"Apply ICA for independent components\"\"\"\n        n_ica_components = min(self.n_components // 2, features.shape[1])\n        \n        if fit:\n            self.ica = FastICA(n_components=n_ica_components, \n                              random_state=self.random_state, \n                              max_iter=1000)\n            ica_features = self.ica.fit_transform(features)\n        else:\n            ica_features = self.ica.transform(features)\n        \n        return ica_features\n    \n    def engineer_features(self, predictions: Dict[str, np.ndarray], fit: bool = True) -> np.ndarray:\n        \"\"\"Create comprehensive non-collinear features\"\"\"\n        print(\"\\nEngineering non-collinear features...\")\n        \n        all_features = {}\n        \n        # 1. Difference features (inherently less correlated)\n        diff_features = self.create_difference_features(predictions)\n        all_features.update(diff_features)\n        print(f\"  Created {len(diff_features)} difference features\")\n        \n        # 2. Rank deviation features\n        rank_features = self.create_rank_deviation_features(predictions)\n        all_features.update(rank_features)\n        print(f\"  Created {len(rank_features)} rank deviation features\")\n        \n        # 3. Non-linear transforms\n        nonlinear_features = self.create_nonlinear_transforms(predictions)\n        all_features.update(nonlinear_features)\n        print(f\"  Created {len(nonlinear_features)} non-linear features\")\n        \n        # 4. Local statistics\n        local_features = self.create_local_statistics(predictions)\n        all_features.update(local_features)\n        print(f\"  Created {len(local_features)} local statistics features\")\n        \n        # Store feature names\n        if fit:\n            self.feature_names = list(all_features.keys())\n        \n        # Convert to array\n        feature_array = np.column_stack([all_features[name] for name in self.feature_names])\n        \n        # 5. Apply transformations to decorrelate\n        if fit:\n            # Standardize\n            feature_array = self.scaler.fit_transform(feature_array)\n            \n            # Apply PCA\n            pca_features = self.apply_pca_transformation(feature_array, fit=True)\n            \n            # Apply ICA for additional independence\n            ica_features = self.apply_ica_transformation(feature_array, fit=True)\n            \n            # Combine PCA and ICA features\n            final_features = np.hstack([pca_features, ica_features])\n            \n            # Apply quantile transformation for better distribution\n            final_features = self.quantile_transformer.fit_transform(final_features)\n        else:\n            # Transform only\n            feature_array = self.scaler.transform(feature_array)\n            pca_features = self.apply_pca_transformation(feature_array, fit=False)\n            ica_features = self.apply_ica_transformation(feature_array, fit=False)\n            final_features = np.hstack([pca_features, ica_features])\n            final_features = self.quantile_transformer.transform(final_features)\n        \n        print(f\"  Final feature dimension: {final_features.shape[1]}\")\n        \n        # Check correlation of final features\n        if fit:\n            corr_matrix = np.corrcoef(final_features.T)\n            np.fill_diagonal(corr_matrix, 0)\n            max_corr = np.max(np.abs(corr_matrix))\n            avg_corr = np.mean(np.abs(corr_matrix))\n            print(f\"  Max correlation in final features: {max_corr:.3f}\")\n            print(f\"  Avg correlation in final features: {avg_corr:.3f}\")\n        \n        return final_features\n\n# =============================================================================\n# Stable Meta-Learners\n# =============================================================================\n\nclass StableStackingEnsemble:\n    \"\"\"Stacking ensemble using non-collinear features\"\"\"\n    \n    def __init__(self, n_folds: int = 5, random_state: int = 42):\n        self.n_folds = n_folds\n        self.random_state = random_state\n        self.feature_engineer = NonCollinearFeatureEngineer(n_components=20, random_state=random_state)\n        self.base_models = {}\n        self.meta_model = None\n        self.oof_predictions = None\n        \n    def _create_base_models(self):\n        \"\"\"Create diverse base models for stacking\"\"\"\n        try:\n            import lightgbm as lgb\n            self.base_models = {\n                'ridge_strong': Ridge(alpha=10.0, random_state=self.random_state),\n                'ridge_weak': Ridge(alpha=0.1, random_state=self.random_state),\n                'elastic': ElasticNet(alpha=1.0, l1_ratio=0.5, random_state=self.random_state),\n                'huber': HuberRegressor(epsilon=1.35, alpha=0.1),\n                'rf': RandomForestRegressor(\n                    n_estimators=100,\n                    max_depth=5,\n                    min_samples_split=50,\n                    random_state=self.random_state\n                ),\n                'et': ExtraTreesRegressor(\n                    n_estimators=100,\n                    max_depth=5,\n                    min_samples_split=50,\n                    random_state=self.random_state\n                ),\n                'lgb': lgb.LGBMRegressor(\n                    n_estimators=100,\n                    num_leaves=31,\n                    learning_rate=0.1,\n                    feature_fraction=0.8,\n                    bagging_fraction=0.8,\n                    random_state=self.random_state,\n                    verbose=-1\n                )\n            }\n        except ImportError:\n            # Fallback without LightGBM\n            self.base_models = {\n                'ridge_strong': Ridge(alpha=10.0, random_state=self.random_state),\n                'ridge_weak': Ridge(alpha=0.1, random_state=self.random_state),\n                'elastic': ElasticNet(alpha=1.0, l1_ratio=0.5, random_state=self.random_state),\n                'huber': HuberRegressor(epsilon=1.35, alpha=0.1),\n                'rf': RandomForestRegressor(n_estimators=100, max_depth=5, random_state=self.random_state),\n                'et': ExtraTreesRegressor(n_estimators=100, max_depth=5, random_state=self.random_state)\n            }\n    \n    def fit(self, predictions: Dict[str, np.ndarray], y: np.ndarray):\n        \"\"\"Fit stacking ensemble with cross-validation\"\"\"\n        print(\"\\nTraining Stable Stacking Ensemble...\")\n        \n        # Create non-collinear features\n        X = self.feature_engineer.engineer_features(predictions, fit=True)\n        \n        # Initialize models\n        self._create_base_models()\n        \n        # Cross-validation for level-1 predictions\n        kf = KFold(n_splits=self.n_folds, shuffle=True, random_state=self.random_state)\n        \n        # Store OOF predictions for each base model\n        self.oof_predictions = np.zeros((len(X), len(self.base_models)))\n        \n        for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n            X_train, X_val = X[train_idx], X[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n            \n            print(f\"  Fold {fold + 1}/{self.n_folds}\")\n            \n            # Train each base model\n            fold_models = {}\n            for i, (name, model) in enumerate(self.base_models.items()):\n                # Clone model for this fold\n                fold_model = model.__class__(**model.get_params())\n                fold_model.fit(X_train, y_train)\n                \n                # Store OOF predictions\n                self.oof_predictions[val_idx, i] = fold_model.predict(X_val)\n                \n                # Store model\n                fold_models[name] = fold_model\n            \n            # Store fold models (we'll retrain on full data later)\n            if fold == 0:\n                self.fold_models = fold_models\n        \n        # Train meta-model on OOF predictions\n        print(\"  Training meta-model on OOF predictions...\")\n        self.meta_model = Ridge(alpha=1.0, random_state=self.random_state)\n        self.meta_model.fit(self.oof_predictions, y)\n        \n        # Retrain base models on full data\n        print(\"  Retraining base models on full data...\")\n        for name, model in self.base_models.items():\n            model.fit(X, y)\n        \n        # Evaluate\n        meta_pred = self.meta_model.predict(self.oof_predictions)\n        score = pearsonr(meta_pred, y)[0]\n        print(f\"  Stacking CV score: {score:.4f}\")\n    \n    def predict(self, predictions: Dict[str, np.ndarray]) -> np.ndarray:\n        \"\"\"Make predictions using stacking ensemble\"\"\"\n        # Create features\n        X = self.feature_engineer.engineer_features(predictions, fit=False)\n        \n        # Get base model predictions\n        base_predictions = np.zeros((len(X), len(self.base_models)))\n        for i, (name, model) in enumerate(self.base_models.items()):\n            base_predictions[:, i] = model.predict(X)\n        \n        # Meta-model prediction\n        return self.meta_model.predict(base_predictions)\n\nclass AdaptiveWeightingEnsemble:\n    \"\"\"Adaptive weighting based on local model performance\"\"\"\n    \n    def __init__(self, lookback_pct: float = 0.2, min_weight: float = 0.05, random_state: int = 42):\n        self.lookback_pct = lookback_pct\n        self.min_weight = min_weight\n        self.random_state = random_state\n        self.base_weights = None\n        self.feature_engineer = NonCollinearFeatureEngineer(n_components=10, random_state=random_state)\n        \n    def _calculate_local_performance(self, predictions: Dict[str, np.ndarray], y: np.ndarray) -> np.ndarray:\n        \"\"\"Calculate rolling performance for each model\"\"\"\n        n_samples = len(y)\n        window_size = max(20, int(n_samples * self.lookback_pct))\n        n_models = len(predictions)\n        \n        # Initialize performance matrix\n        performance = np.zeros((n_samples, n_models))\n        \n        # Calculate rolling correlations\n        for i, (name, pred) in enumerate(predictions.items()):\n            for j in range(n_samples):\n                start = max(0, j - window_size)\n                end = j + 1\n                \n                if end - start > 10:  # Minimum samples\n                    try:\n                        corr = pearsonr(pred[start:end], y[start:end])[0]\n                        if np.isnan(corr):\n                            corr = 0.5\n                    except:\n                        corr = 0.5\n                else:\n                    corr = 0.5\n                \n                performance[j, i] = max(0, corr)  # Clip negative correlations\n        \n        return performance\n    \n    def fit(self, predictions: Dict[str, np.ndarray], y: np.ndarray):\n        \"\"\"Fit adaptive weighting model\"\"\"\n        print(\"\\nTraining Adaptive Weighting Ensemble...\")\n        \n        # Calculate local performance\n        performance = self._calculate_local_performance(predictions, y)\n        \n        # Calculate base weights (global performance)\n        self.base_weights = np.zeros(len(predictions))\n        for i, (name, pred) in enumerate(predictions.items()):\n            self.base_weights[i] = max(0, pearsonr(pred, y)[0])\n        \n        # Normalize base weights\n        if np.sum(self.base_weights) > 0:\n            self.base_weights = self.base_weights / np.sum(self.base_weights)\n        else:\n            self.base_weights = np.ones(len(predictions)) / len(predictions)\n        \n        print(\"  Base weights:\")\n        for name, weight in zip(predictions.keys(), self.base_weights):\n            print(f\"    {name}: {weight:.3f}\")\n        \n        # Store performance history\n        self.performance_history = performance\n        self.model_names = list(predictions.keys())\n    \n    def predict(self, predictions: Dict[str, np.ndarray]) -> np.ndarray:\n        \"\"\"Make predictions with adaptive weights\"\"\"\n        pred_array = np.column_stack(list(predictions.values()))\n        \n        # Use base weights (in production, you could adapt these based on recent performance)\n        weights = self.base_weights\n        \n        # Ensure minimum weight\n        weights = np.maximum(weights, self.min_weight)\n        weights = weights / np.sum(weights)\n        \n        # Weighted average\n        return np.average(pred_array, axis=1, weights=weights)\n\n# =============================================================================\n# Main Stable Meta-Learning Ensemble\n# =============================================================================\n\nclass StableMetaLearningEnsemble:\n    \"\"\"Main ensemble using stable, non-collinear meta-learning\"\"\"\n    \n    def __init__(self):\n        # Base configuration\n        self.model_name = \"stable_meta_ensemble\"\n        self.model_directory = os.path.join(global_config.base_dir, \"stable_meta_ensemble\")\n        \n        # Register with global configuration\n        global_config.register_model(self.model_name, self.model_directory)\n        \n        # Output paths\n        self.final_submission_path = \"/kaggle/working/stable_final_submission.csv\"\n        self.ensemble_analysis_path = os.path.join(self.model_directory, \"ensemble_analysis.csv\")\n        self.meta_weights_path = os.path.join(self.model_directory, \"meta_weights.json\")\n        self.visualization_path = os.path.join(self.model_directory, \"ensemble_visualization.png\")\n        \n        # Initialize meta-learners\n        self.stacking = StableStackingEnsemble(n_folds=5)\n        self.adaptive = AdaptiveWeightingEnsemble(lookback_pct=0.2)\n        \n        # Ensure directories exist\n        Path(self.model_directory).mkdir(parents=True, exist_ok=True)\n    \n    def load_base_predictions(self) -> Tuple[Dict[str, pd.DataFrame], Dict[str, np.ndarray]]:\n        \"\"\"Load predictions from all completed models\"\"\"\n        submissions = {}\n        predictions = {}\n        \n        print(\"\\nLoading base model predictions...\")\n        \n        # Reload state\n        global_config.load_state()\n        \n        for model_name, model_record in global_config.model_registry.items():\n            if model_name == self.model_name or model_record.status != \"completed\":\n                continue\n            \n            # Find submission file\n            submission_path = None\n            for output in model_record.outputs.values():\n                if output.file_type == 'submission' and os.path.exists(output.file_path):\n                    submission_path = output.file_path\n                    break\n            \n            if submission_path:\n                try:\n                    df = pd.read_csv(submission_path)\n                    if df.shape[1] >= 2:\n                        submissions[model_name] = df\n                        predictions[model_name] = df.iloc[:, 1].values\n                        print(f\"  ✓ Loaded {model_name}\")\n                except Exception as e:\n                    print(f\"  ✗ Error loading {model_name}: {e}\")\n        \n        print(f\"\\nLoaded {len(predictions)} models successfully\")\n        return submissions, predictions\n    \n    def create_visualization(self, predictions: Dict[str, np.ndarray], \n                           final_predictions: np.ndarray, synthetic_y: np.ndarray):\n        \"\"\"Create visualization of ensemble results\"\"\"\n        try:\n            fig, axes = plt.subplots(2, 2, figsize=(12, 10))\n            \n            # 1. Model correlations heatmap\n            ax = axes[0, 0]\n            pred_array = np.column_stack(list(predictions.values()))\n            corr_matrix = np.corrcoef(pred_array.T)\n            im = ax.imshow(corr_matrix, cmap='coolwarm', vmin=-1, vmax=1)\n            ax.set_title('Base Model Correlations')\n            ax.set_xticks(range(len(predictions)))\n            ax.set_yticks(range(len(predictions)))\n            ax.set_xticklabels(list(predictions.keys()), rotation=45, ha='right')\n            ax.set_yticklabels(list(predictions.keys()))\n            plt.colorbar(im, ax=ax)\n            \n            # 2. Prediction distributions\n            ax = axes[0, 1]\n            for name, pred in predictions.items():\n                ax.hist(pred, bins=50, alpha=0.3, label=name, density=True)\n            ax.hist(final_predictions, bins=50, alpha=0.8, label='Final', \n                   density=True, color='black', histtype='step', linewidth=2)\n            ax.set_xlabel('Prediction Value')\n            ax.set_ylabel('Density')\n            ax.set_title('Prediction Distributions')\n            ax.legend()\n            \n            # 3. Model weights from adaptive ensemble\n            ax = axes[1, 0]\n            if hasattr(self.adaptive, 'base_weights'):\n                model_names = list(predictions.keys())\n                weights = self.adaptive.base_weights\n                bars = ax.bar(range(len(weights)), weights)\n                ax.set_xticks(range(len(weights)))\n                ax.set_xticklabels(model_names, rotation=45, ha='right')\n                ax.set_ylabel('Weight')\n                ax.set_title('Adaptive Ensemble Weights')\n                \n                # Add value labels\n                for i, (bar, weight) in enumerate(zip(bars, weights)):\n                    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01, \n                           f'{weight:.3f}', ha='center', va='bottom')\n            \n            # 4. Final vs synthetic target\n            ax = axes[1, 1]\n            ax.scatter(synthetic_y[::10], final_predictions[::10], alpha=0.5, s=1)\n            ax.plot([synthetic_y.min(), synthetic_y.max()], \n                   [synthetic_y.min(), synthetic_y.max()], 'r--', lw=2)\n            ax.set_xlabel('Synthetic Target')\n            ax.set_ylabel('Final Predictions')\n            ax.set_title('Meta-Learning Fit')\n            corr = pearsonr(synthetic_y, final_predictions)[0]\n            ax.text(0.05, 0.95, f'Correlation: {corr:.4f}', \n                   transform=ax.transAxes, verticalalignment='top')\n            \n            plt.suptitle('Stable Meta-Learning Ensemble Analysis', fontsize=14)\n            plt.tight_layout()\n            plt.savefig(self.visualization_path, dpi=150, bbox_inches='tight')\n            plt.close()\n            \n            print(f\"\\nVisualization saved to: {self.visualization_path}\")\n            \n            global_config.register_model_output(\n                self.model_name,\n                'visualization',\n                self.visualization_path,\n                'visualization'\n            )\n            \n        except Exception as e:\n            print(f\"Warning: Could not create visualization: {e}\")\n    \n    def run(self) -> pd.DataFrame:\n        \"\"\"Execute the stable meta-learning ensemble pipeline\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"STABLE META-LEARNING ENSEMBLE (NON-COLLINEAR)\")\n        print(\"=\"*80)\n        \n        # Update status\n        global_config.update_model_status(self.model_name, 'running')\n        \n        try:\n            # Load base predictions\n            submissions, predictions = self.load_base_predictions()\n            \n            if len(predictions) < 2:\n                raise ValueError(f\"Insufficient models for ensemble: {len(predictions)}\")\n            \n            # Create synthetic labels (weighted by diversity)\n            print(\"\\nCreating synthetic labels...\")\n            pred_array = np.column_stack(list(predictions.values()))\n            \n            # Use rank-based weighting for robustness\n            rank_array = np.column_stack([rankdata(pred) for pred in predictions.values()])\n            rank_correlations = np.corrcoef(rank_array.T)\n            \n            # Weight by uniqueness (lower correlation = higher weight)\n            uniqueness = 1 - (np.sum(np.abs(rank_correlations), axis=1) - 1) / (len(predictions) - 1)\n            weights = uniqueness / np.sum(uniqueness)\n            \n            synthetic_y = np.average(pred_array, axis=1, weights=weights)\n            \n            # Add small noise\n            np.random.seed(42)\n            noise = np.random.normal(0, np.std(synthetic_y) * 0.005, size=len(synthetic_y))\n            synthetic_y += noise\n            \n            print(f\"  Synthetic target stats: mean={np.mean(synthetic_y):.6f}, std={np.std(synthetic_y):.6f}\")\n            \n            # Train meta-learners\n            print(\"\\nTraining meta-learners...\")\n            \n            # 1. Stacking ensemble\n            self.stacking.fit(predictions, synthetic_y)\n            stacking_pred = self.stacking.predict(predictions)\n            \n            # 2. Adaptive weighting\n            self.adaptive.fit(predictions, synthetic_y)\n            adaptive_pred = self.adaptive.predict(predictions)\n            \n            # 3. Simple robust average (median of predictions)\n            robust_pred = np.median(list(predictions.values()), axis=0)\n            \n            # Final blend\n            print(\"\\nCreating final blend...\")\n            final_predictions = (\n                0.5 * stacking_pred +\n                0.3 * adaptive_pred +\n                0.2 * robust_pred\n            )\n            \n            # Post-processing\n            print(\"\\nApplying post-processing...\")\n            \n            # Clip to reasonable bounds\n            all_preds = np.concatenate(list(predictions.values()))\n            lower_bound = np.percentile(all_preds, 0.1)\n            upper_bound = np.percentile(all_preds, 99.9)\n            final_predictions = np.clip(final_predictions, lower_bound, upper_bound)\n            \n            # Smooth extreme outliers\n            z_scores = np.abs((final_predictions - np.median(final_predictions)) / \n                            (1.4826 * np.median(np.abs(final_predictions - np.median(final_predictions)))))\n            outlier_mask = z_scores > 4\n            if np.sum(outlier_mask) > 0:\n                print(f\"  Smoothing {np.sum(outlier_mask)} outliers\")\n                final_predictions[outlier_mask] = (\n                    0.7 * np.median(final_predictions) + \n                    0.3 * final_predictions[outlier_mask]\n                )\n            \n            # Create submission\n            template = next(iter(submissions.values()))\n            final_submission = template.copy()\n            final_submission.iloc[:, 1] = final_predictions\n            \n            # Save outputs\n            final_submission.to_csv(self.final_submission_path, index=False)\n            print(f\"\\nFinal submission saved to: {self.final_submission_path}\")\n            \n            global_config.register_model_output(\n                self.model_name,\n                'final_submission',\n                self.final_submission_path,\n                'submission'\n            )\n            \n            # Save meta information\n            meta_info = {\n                'n_base_models': len(predictions),\n                'base_models': list(predictions.keys()),\n                'ensemble_weights': {\n                    'stacking': 0.5,\n                    'adaptive': 0.3,\n                    'robust': 0.2\n                },\n                'adaptive_weights': dict(zip(predictions.keys(), \n                                           self.adaptive.base_weights.tolist())),\n                'prediction_stats': {\n                    'mean': float(np.mean(final_predictions)),\n                    'std': float(np.std(final_predictions)),\n                    'min': float(np.min(final_predictions)),\n                    'max': float(np.max(final_predictions))\n                }\n            }\n            \n            with open(self.meta_weights_path, 'w') as f:\n                json.dump(meta_info, f, indent=2)\n            \n            # Create visualization\n            self.create_visualization(predictions, final_predictions, synthetic_y)\n            \n            # Update status\n            global_config.update_model_status(self.model_name, 'completed', \n                                            score=pearsonr(final_predictions, synthetic_y)[0])\n            \n            print(\"\\n✅ Stable meta-learning ensemble completed successfully\")\n            \n            return final_submission\n            \n        except Exception as e:\n            error_msg = str(e)\n            global_config.update_model_status(self.model_name, 'failed', error_message=error_msg)\n            print(f\"\\n❌ Stable meta-learning ensemble failed: {error_msg}\")\n            raise\n\n# =============================================================================\n# Main Execution\n# =============================================================================\n\ndef create_stable_meta_ensemble():\n    \"\"\"Create the final ensemble using stable meta-learning\"\"\"\n    print(\"\\nDRW Crypto Market Prediction - Stable Meta-Learning Ensemble\")\n    print(\"=\"*80)\n    \n    try:\n        # Clean memory\n        gc.collect()\n        \n        # Create ensemble\n        ensemble = StableMetaLearningEnsemble()\n        \n        # Run pipeline\n        final_submission = ensemble.run()\n        \n        return final_submission\n        \n    except Exception as e:\n        print(f\"\\nError: {e}\")\n        raise\n\n# Entry point\nif __name__ == \"__main__\":\n    final_submission = create_stable_meta_ensemble()\n    \n    # Display summary\n    print(\"\\n\" + \"=\"*80)\n    print(\"Pipeline Execution Summary:\")\n    print(global_config.get_execution_summary())\n    print(\"=\"*80)","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}