{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"},{"sourceId":12307790,"sourceType":"datasetVersion","datasetId":7757718}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python3\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch.cuda.amp import GradScaler, autocast\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer, PowerTransformer, MinMaxScaler\nfrom sklearn.model_selection import KFold, TimeSeriesSplit, StratifiedKFold\nfrom sklearn.decomposition import PCA, TruncatedSVD\nfrom sklearn.cluster import KMeans\nfrom sklearn.ensemble import RandomForestRegressor\nfrom scipy.stats import pearsonr, spearmanr, rankdata\nfrom scipy.special import expit, logit\nimport warnings\nimport gc\nimport psutil\nimport time\nfrom typing import List, Dict, Tuple, Optional, Set, Any, Union\nfrom collections import defaultdict\nimport random\nimport json\nimport hashlib\nfrom datetime import datetime, timedelta\nimport os\nimport itertools\nfrom dataclasses import dataclass, field\nimport traceback\nimport signal\nimport sys\nfrom concurrent.futures import ThreadPoolExecutor, TimeoutError\nimport threading\nimport math\nimport pickle\n\ntry:\n    import gspread\n    from google.oauth2.service_account import Credentials\n    GSPREAD_AVAILABLE = True\nexcept ImportError:\n    GSPREAD_AVAILABLE = False\n    print(\"Warning: Google Sheets libraries not available. Install with: pip install gspread google-auth\")\n\nwarnings.filterwarnings('ignore')\n\ndef to_python_type(obj):\n    \"\"\"Convert numpy types to Python native types for JSON serialization\"\"\"\n    if isinstance(obj, np.integer):\n        return int(obj)\n    elif isinstance(obj, np.floating):\n        return float(obj)\n    elif isinstance(obj, np.ndarray):\n        return obj.tolist()\n    elif torch is not None and isinstance(obj, torch.Tensor):\n        return obj.cpu().numpy().tolist()\n    elif isinstance(obj, (np.bool_, bool)):\n        return bool(obj)\n    elif isinstance(obj, list):\n        return [to_python_type(item) for item in obj]\n    elif isinstance(obj, dict):\n        return {key: to_python_type(value) for key, value in obj.items()}\n    else:\n        return obj\n\ndef make_hashable(obj):\n    \"\"\"Recursively convert any object to a hashable format\"\"\"\n    if isinstance(obj, dict):\n        # Sort dict items and convert to tuple of tuples\n        return tuple(sorted((k, make_hashable(v)) for k, v in obj.items()))\n    elif isinstance(obj, list):\n        # Convert list to tuple\n        return tuple(make_hashable(item) for item in obj)\n    elif isinstance(obj, set):\n        # Convert set to sorted tuple\n        return tuple(sorted(make_hashable(item) for item in obj))\n    elif isinstance(obj, (str, int, float, bool, type(None))):\n        # These are already hashable\n        return obj\n    elif isinstance(obj, tuple):\n        # Make sure all elements in tuple are hashable\n        return tuple(make_hashable(item) for item in obj)\n    else:\n        # For any other type, convert to string\n        return str(obj)\n\n@dataclass\nclass FeatureGroupConfig:\n    market_features: List[str] = field(default_factory=lambda: [\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n    ])\n    \n    microstructure_features: List[str] = field(default_factory=lambda: [\n        \"volume_weighted_sell\", \"buy_sell_ratio\", \n        \"selling_pressure\", \"effective_spread_proxy\",\n        \"bid_ask_imbalance\", \"flow_toxicity\",\n        \"volume_concentration\", \"liquidity_consumption\",\n        \"price_pressure\", \"order_imbalance\", \"relative_spread\"\n    ])\n    \n    core_proprietary_features: List[str] = field(default_factory=lambda: [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \n        \"X674\", \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \n        \"X612\", \"X888\", \"X421\", \"X333\", \"X292\"\n    ])\n    \n    dropout_rates: Dict[str, float] = field(default_factory=lambda: {\n        'market': 0.3,\n        'microstructure': 0.4,\n        'core_proprietary': 0.5,\n        'other_proprietary': 0.6,\n        'interactions': 0.4\n    })\n    \n    noise_levels: Dict[str, float] = field(default_factory=lambda: {\n        'market': 0.002,\n        'microstructure': 0.003,\n        'core_proprietary': 0.006,\n        'other_proprietary': 0.015,\n        'interactions': 0.003\n    })\n    \n    def get_feature_indices(self, all_features: List[str]) -> Dict[str, List[int]]:\n        indices = {}\n        \n        indices['market'] = [\n            i for i, f in enumerate(all_features) \n            if f in self.market_features\n        ]\n        \n        indices['microstructure'] = [\n            i for i, f in enumerate(all_features) \n            if f in self.microstructure_features\n        ]\n        \n        indices['core_proprietary'] = [\n            i for i, f in enumerate(all_features) \n            if f in self.core_proprietary_features\n        ]\n        \n        indices['other_proprietary'] = [\n            i for i, f in enumerate(all_features) \n            if f.startswith('X') and f not in self.core_proprietary_features\n        ]\n        \n        return indices\n\ndef create_advanced_microstructure_features(df: pd.DataFrame) -> pd.DataFrame:\n    df = df.copy()\n    \n    # Add small epsilon to avoid division by zero\n    eps = 1e-8\n    \n    df['volume_weighted_sell'] = df['sell_qty'].values * df['volume'].values\n    df['buy_sell_ratio'] = df['buy_qty'].values / (df['sell_qty'].values + eps)\n    df['selling_pressure'] = df['sell_qty'].values / (df['volume'].values + eps)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'].values - df['sell_qty'].values) / (df['volume'].values + eps)\n    \n    df['bid_ask_imbalance'] = (\n        (df['bid_qty'] - df['ask_qty']) / \n        (df['bid_qty'] + df['ask_qty'] + eps)\n    )\n    \n    df['flow_toxicity'] = df['sell_qty'] / (df['buy_qty'] + eps)\n    \n    df['volume_concentration'] = (\n        (df['buy_qty'] + df['sell_qty']) / (df['volume'] + eps)\n    )\n    \n    df['liquidity_consumption'] = (\n        (df['buy_qty'] + df['sell_qty']) / \n        (df['bid_qty'] + df['ask_qty'] + eps)\n    )\n    \n    df['price_pressure'] = (\n        (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    )\n    \n    df['order_imbalance'] = (\n        df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + eps)\n    )\n    \n    df['relative_spread'] = (\n        np.abs(df['buy_qty'] - df['sell_qty']) / \n        (df['bid_qty'] + df['ask_qty'] + eps)\n    )\n    \n    # Handle infinities and NaNs\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    return df\n\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    CREDENTIALS_FILE = '/kaggle/input/private-data-set-model-tracking-tool/forward-leaf-464213-u2-489add6f6da1.json'\n    SPREADSHEET_URL = 'https://docs.google.com/spreadsheets/d/1u9rj-YXCgriK3ydSHkiZPSNTjSBE8XxEdV6vpNGUMWo/edit?gid=0#gid=0'\n    \n    FEATURE_GROUP_CONFIG = FeatureGroupConfig()\n    \n    CORE_FEATURES = (\n        FEATURE_GROUP_CONFIG.market_features +\n        FEATURE_GROUP_CONFIG.microstructure_features +\n        FEATURE_GROUP_CONFIG.core_proprietary_features\n    )\n    \n    # More conservative baseline\n    BASELINE_LAYERS = [256, 64, 1]\n    BASELINE_DROPOUT = 0.7\n    BASELINE_NOISE = 0.01\n    \n    MAX_RUNTIME_HOURS = 8.0\n    CHECKPOINT_INTERVAL_MINUTES = 30\n    \n    MIN_MEMORY_GB = 2.0\n    MEMORY_CHECK_INTERVAL = 10\n    \n    MAX_ADDITIONAL_FEATURES = 3\n    \n    # USE ALL DATA FOR TRAINING!\n    USE_FULL_DATASET = True  # Changed from sampling\n    SAMPLE_SIZE_FOR_TESTING = None  # No sampling\n    \n    BATCH_SIZE = 1024  # Reduced for better generalization\n    MAX_EPOCHS = 20    # Reduced to prevent overfitting\n    EARLY_STOPPING_PATIENCE = 3  # More aggressive early stopping\n    \n    N_FOLDS = 5  # More folds for better validation\n    \n    INITIAL_RANDOM_CONFIGS = 30  # Fewer initial configs\n    EXPLOITATION_RATE = 0.7\n    EXPLORATION_RATE = 0.3\n    \n    # Conservative overfitting thresholds\n    MAX_TRAIN_VAL_GAP = 0.05  # Maximum acceptable train-val gap\n    MIN_ACCEPTABLE_VAL_SCORE = 0.0  # Minimum acceptable validation score\n    \n    INFINITY_STRATEGIES = ['median', 'percentile', 'zero']\n    \n    FEATURE_TRANSFORMS = [\n        ['standard'],\n        ['rank'],\n        ['quantile'],\n        ['standard', 'rank'],\n        ['robust']\n    ]\n    \n    LABEL_TRANSFORMS = ['none', 'rank', 'quantile']\n    \n    ARCHITECTURE_VARIANTS = [\n        {'name': 'baseline_mlp', 'hidden_dims': [256, 64, 1], 'type': 'standard'},\n        {'name': 'conservative', 'hidden_dims': [128, 32, 1], 'type': 'standard'},\n        {'name': 'moderate', 'hidden_dims': [256, 128, 64, 1], 'type': 'standard'},\n        {'name': 'wide_shallow', 'hidden_dims': [512, 1], 'type': 'wide'},\n        {'name': 'pyramid', 'hidden_dims': [256, 128, 64, 32, 1], 'type': 'pyramid'}\n    ]\n    \n    ACTIVATIONS = ['relu', 'tanh', 'leaky_relu']\n    \n    DROPOUT_RATES = [0.5, 0.6, 0.7, 0.8]  # Higher dropout rates\n    \n    LEARNING_RATES = [0.0001, 0.0005, 0.001]  # More conservative\n    \n    WEIGHT_DECAYS = [0.001, 0.01, 0.1]  # Higher regularization\n    \n    INPUT_NOISE_LEVELS = [0.01, 0.02, 0.05]  # More noise\n    \n    CV_STRATEGIES = ['kfold', 'timeseries_gap', 'stratified']\n    \n    MAX_CONFIGS_PER_FEATURE = 50\n\n@dataclass\nclass ExperimentConfig:\n    feature_list: List[str]\n    additional_features: List[str]\n    \n    infinity_strategy: str = 'median'\n    feature_transforms: List[str] = field(default_factory=lambda: ['standard'])\n    use_interactions: bool = False\n    interaction_features: List[str] = field(default_factory=list)\n    interaction_method: str = 'multiply'\n    use_clustering: bool = False\n    n_clusters: int = 10\n    use_pca: bool = False\n    pca_components: int = 20\n    use_binning: bool = False\n    binning_method: str = 'quantile'\n    binning_features: List[str] = field(default_factory=list)\n    n_bins: int = 10\n    \n    label_transform: str = 'none'\n    label_noise: float = 0.0\n    \n    architecture_name: str = 'medium'\n    architecture_type: str = 'standard'\n    hidden_dims: List[int] = field(default_factory=lambda: [256, 128, 64])\n    activation: str = 'relu'\n    dropout_rate: float = 0.6\n    dropout_rates_dict: Dict[str, float] = field(default_factory=dict)\n    noise_levels_dict: Dict[str, float] = field(default_factory=dict)\n    use_batch_norm: bool = False\n    use_layer_norm: bool = False\n    use_residual: bool = False\n    \n    optimizer: str = 'adam'\n    learning_rate: float = 0.001\n    weight_decay: float = 0.01\n    \n    input_noise: float = 0.02\n    use_mixup: bool = False\n    mixup_alpha: float = 0.2\n    use_cutmix: bool = False\n    cutmix_alpha: float = 1.0\n    gradient_clip: float = 1.0\n    \n    cv_strategy: str = 'kfold'\n    n_ensemble: int = 1\n    \n    def get_hash(self) -> str:\n        config_dict = {\n            'features': sorted(self.additional_features),\n            'infinity': self.infinity_strategy,\n            'transforms': sorted(self.feature_transforms),\n            'interactions': self.use_interactions,\n            'interaction_features': sorted(self.interaction_features) if self.interaction_features else [],\n            'interaction_method': self.interaction_method,\n            'clustering': self.use_clustering,\n            'n_clusters': self.n_clusters,\n            'pca': self.use_pca,\n            'pca_components': self.pca_components,\n            'binning': self.use_binning,\n            'binning_method': self.binning_method,\n            'binning_features': sorted(self.binning_features) if self.binning_features else [],\n            'label_transform': self.label_transform,\n            'label_noise': self.label_noise,\n            'architecture': self.architecture_name,\n            'hidden_dims': self.hidden_dims,\n            'activation': self.activation,\n            'dropout': self.dropout_rate,\n            'batch_norm': self.use_batch_norm,\n            'layer_norm': self.use_layer_norm,\n            'residual': self.use_residual,\n            'optimizer': self.optimizer,\n            'lr': self.learning_rate,\n            'weight_decay': self.weight_decay,\n            'input_noise': self.input_noise,\n            'mixup': self.use_mixup,\n            'cutmix': self.use_cutmix,\n            'cv_strategy': self.cv_strategy,\n            'n_ensemble': self.n_ensemble\n        }\n        config_str = json.dumps(config_dict, sort_keys=True)\n        return hashlib.sha256(config_str.encode()).hexdigest()[:16]\n    \n    def get_description(self) -> str:\n        parts = []\n        \n        feature_group_config = Config.FEATURE_GROUP_CONFIG\n        \n        n_market = len([f for f in self.feature_list if f in feature_group_config.market_features])\n        n_micro = len([f for f in self.feature_list if f in feature_group_config.microstructure_features])\n        n_core_prop = len([f for f in self.feature_list if f in feature_group_config.core_proprietary_features])\n        n_other_prop = len([f for f in self.additional_features if f.startswith('X')])\n        \n        parts.append(f\"M:{n_market} Mi:{n_micro} CP:{n_core_prop} OP:{n_other_prop}\")\n        if self.additional_features:\n            parts.append(f\"Add:{self.additional_features[:2]}{'...' if len(self.additional_features) > 2 else ''}\")\n        parts.append(f\"Inf:{self.infinity_strategy[:3]}\")\n        parts.append(f\"Arch:{self.architecture_name}\")\n        parts.append(f\"Drop:{self.dropout_rate}\")\n        parts.append(f\"LR:{self.learning_rate}\")\n        \n        return \" | \".join(parts)\n\ndef create_baseline_config() -> ExperimentConfig:\n    return ExperimentConfig(\n        feature_list=Config.CORE_FEATURES,\n        additional_features=[],\n        architecture_name='baseline_mlp',\n        architecture_type='standard',\n        hidden_dims=[256, 64, 1],\n        activation='relu',\n        dropout_rate=0.7,\n        learning_rate=0.001,\n        weight_decay=0.01,\n        input_noise=0.01,\n        optimizer='adam',\n        use_batch_norm=False,\n        use_layer_norm=False,\n        label_transform='none',\n        cv_strategy='timeseries_gap',\n        n_ensemble=1,\n        infinity_strategy='median',\n        feature_transforms=['standard']\n    )\n\nclass MemoryManager:\n    def __init__(self):\n        self.gpu_available = torch.cuda.is_available()\n        \n    def get_memory_info(self) -> Dict[str, float]:\n        info = {\n            'cpu_percent': psutil.virtual_memory().percent,\n            'cpu_available_gb': psutil.virtual_memory().available / 1e9,\n            'cpu_used_gb': psutil.virtual_memory().used / 1e9\n        }\n        \n        if self.gpu_available:\n            for i in range(torch.cuda.device_count()):\n                info[f'gpu_{i}_allocated_gb'] = torch.cuda.memory_allocated(i) / 1e9\n                info[f'gpu_{i}_reserved_gb'] = torch.cuda.memory_reserved(i) / 1e9\n        \n        return info\n    \n    def clear_memory(self):\n        gc.collect()\n        if self.gpu_available:\n            torch.cuda.empty_cache()\n    \n    def check_memory_available(self, required_gb: float = 2.0) -> bool:\n        info = self.get_memory_info()\n        return info['cpu_available_gb'] > required_gb\n\nclass EnhancedSheetsTracker:\n    def __init__(self, credentials_file: str, spreadsheet_url: str):\n        if not GSPREAD_AVAILABLE:\n            raise ImportError(\"gspread not available\")\n        \n        self.gc = gspread.service_account(filename=credentials_file)\n        self.spreadsheet = self.gc.open_by_url(spreadsheet_url)\n        \n        try:\n            self.sheet = self.spreadsheet.worksheet(\"Experiments_Enhanced_V3\")\n        except:\n            self.sheet = self.spreadsheet.add_worksheet(\"Experiments_Enhanced_V3\", 50000, 100)\n        \n        self.columns = [\n            'experiment_id', 'timestamp_start', 'timestamp_end', 'status',\n            'config_hash', 'baseline_similarity_score',\n            \n            'core_features_list', 'n_core_features',\n            'market_features_list', 'n_market_features', \n            'microstructure_features_list', 'n_microstructure_features',\n            'additional_x_features_list', 'n_additional_x_features',\n            'total_features',\n            \n            'core_features_transform', 'market_features_transform',\n            'x_features_transform', 'global_transforms',\n            'infinity_strategy',\n            \n            'use_interactions', 'interaction_pairs', 'n_interactions',\n            'interaction_method', 'interaction_features_source',\n            \n            'mean_train_score', 'mean_val_score', 'std_train_score', 'std_val_score',\n            'train_val_gap', 'best_fold_score', 'worst_fold_score',\n            'improvement_over_baseline', 'overfitting_flag',\n            \n            'architecture_name', 'architecture_category', 'hidden_dims', \n            'total_parameters', 'depth', 'width',\n            \n            'activation', 'dropout_rates_dict', 'optimizer', 'learning_rate',\n            'weight_decay', 'noise_levels_dict', 'batch_size', 'input_noise',\n            \n            'use_clustering', 'n_clusters', 'use_pca', 'pca_components',\n            'use_binning', 'binning_details',\n            \n            'label_transform', 'cv_strategy', 'n_folds', 'n_ensemble',\n            'gradient_clip', 'use_mixup', 'use_cutmix',\n            'use_batch_norm', 'use_layer_norm', 'use_residual',\n            \n            'training_time_minutes', 'memory_peak_gb',\n            'submission_file', 'error_message',\n            \n            'fold_scores_json', 'fold_train_scores_json', 'config_full_json'\n        ]\n        \n        self._ensure_headers()\n        \n        self.config_cache = set()\n        self._load_config_cache()\n        \n        # Track row numbers for each experiment\n        self.experiment_rows = {}\n        \n        self.baseline_score = None\n        \n        # Get the next available row\n        all_values = self.sheet.get_all_values()\n        self.next_row = len(all_values) + 1\n        \n        print(f\"Initialized sheets tracker. Next row: {self.next_row}\")\n        \n    def _ensure_headers(self):\n        try:\n            current_headers = self.sheet.row_values(1)\n            if current_headers != self.columns:\n                self.sheet.update('A1', [self.columns])\n                print(\"Updated spreadsheet headers\")\n        except:\n            self.sheet.update('A1', [self.columns])\n            print(\"Created spreadsheet headers\")\n    \n    def _load_config_cache(self):\n        try:\n            all_values = self.sheet.get_all_values()\n            if len(all_values) > 1:\n                header_row = all_values[0]\n                if 'config_hash' in header_row:\n                    hash_idx = header_row.index('config_hash')\n                    for row in all_values[1:]:\n                        if len(row) > hash_idx and row[hash_idx]:\n                            self.config_cache.add(row[hash_idx])\n            print(f\"Loaded {len(self.config_cache)} existing configurations from cache\")\n        except Exception as e:\n            print(f\"Warning: Failed to load config cache: {e}\")\n    \n    def config_exists(self, config: ExperimentConfig) -> bool:\n        return config.get_hash() in self.config_cache\n    \n    def log_experiment_start(self, experiment_id: str, config: ExperimentConfig) -> int:\n        \"\"\"Log experiment start - writes immediately to spreadsheet\"\"\"\n        config_hash = config.get_hash()\n        \n        row_data = self._prepare_row_data(config, {\n            'experiment_id': experiment_id,\n            'timestamp_start': datetime.now().isoformat(),\n            'timestamp_end': '',\n            'status': 'Started',\n            'config_hash': config_hash,\n            'baseline_similarity': 0\n        })\n        \n        # Create row list with proper order\n        row_list = []\n        for col in self.columns:\n            value = row_data.get(col, '')\n            row_list.append(str(value) if value is not None else '')\n        \n        try:\n            # Write row immediately\n            self.sheet.append_row(row_list, value_input_option='RAW')\n            row_number = self.next_row\n            self.next_row += 1\n            self.experiment_rows[experiment_id] = row_number\n            self.config_cache.add(config_hash)\n            \n            print(f\"✓ Started experiment {experiment_id} at row {row_number}\")\n            return row_number\n            \n        except Exception as e:\n            print(f\"✗ Failed to log experiment start: {e}\")\n            traceback.print_exc()\n            return -1\n    \n    def update_experiment_status(self, experiment_id: str, status: str, results: Dict[str, Any] = None):\n        \"\"\"Update experiment status - writes immediately to spreadsheet\"\"\"\n        if experiment_id not in self.experiment_rows:\n            print(f\"Warning: No row found for experiment {experiment_id}\")\n            return\n        \n        row_number = self.experiment_rows[experiment_id]\n        \n        try:\n            # Build batch update\n            updates = []\n            \n            # Always update status and end time\n            updates.append({\n                'range': f'D{row_number}',  # status column\n                'values': [[status]]\n            })\n            updates.append({\n                'range': f'C{row_number}',  # timestamp_end column\n                'values': [[datetime.now().isoformat()]]\n            })\n            \n            # Update scores if available\n            if results and 'mean_val_score' in results:\n                col_indices = {col: i for i, col in enumerate(self.columns)}\n                \n                # Update validation score\n                val_col = self._get_column_letter(col_indices['mean_val_score'] + 1)\n                updates.append({\n                    'range': f'{val_col}{row_number}',\n                    'values': [[str(round(float(results.get('mean_val_score', 0)), 4))]]\n                })\n                \n                # Update train score\n                if 'mean_train_score' in results:\n                    train_col = self._get_column_letter(col_indices['mean_train_score'] + 1)\n                    updates.append({\n                        'range': f'{train_col}{row_number}',\n                        'values': [[str(round(float(results.get('mean_train_score', 0)), 4))]]\n                    })\n                \n                # Update train-val gap\n                if 'train_val_gap' in results:\n                    gap_col = self._get_column_letter(col_indices['train_val_gap'] + 1)\n                    updates.append({\n                        'range': f'{gap_col}{row_number}',\n                        'values': [[str(round(float(results.get('train_val_gap', 0)), 4))]]\n                    })\n                \n                # Update overfitting flag\n                if 'overfitting_flag' in results:\n                    overfit_col = self._get_column_letter(col_indices['overfitting_flag'] + 1)\n                    updates.append({\n                        'range': f'{overfit_col}{row_number}',\n                        'values': [[str(results.get('overfitting_flag', False))]]\n                    })\n                \n                # Update training time\n                if 'training_time_minutes' in results:\n                    time_col = self._get_column_letter(col_indices['training_time_minutes'] + 1)\n                    updates.append({\n                        'range': f'{time_col}{row_number}',\n                        'values': [[str(round(float(results.get('training_time_minutes', 0)), 2))]]\n                    })\n            \n            # Execute batch update\n            self.sheet.batch_update(updates)\n            print(f\"✓ Updated experiment {experiment_id} status to {status}\")\n            \n        except Exception as e:\n            print(f\"✗ Failed to update experiment status: {e}\")\n            traceback.print_exc()\n    \n    def log_experiment_complete(self, experiment_id: str, config: ExperimentConfig, results: Dict[str, Any]):\n        \"\"\"Update complete experiment results\"\"\"\n        if experiment_id not in self.experiment_rows:\n            print(f\"Warning: No row found for experiment {experiment_id}\")\n            return\n        \n        row_number = self.experiment_rows[experiment_id]\n        \n        # Prepare complete row data\n        row_data = self._prepare_row_data(config, results)\n        \n        # Create row list with proper order\n        row_list = []\n        for col in self.columns:\n            value = row_data.get(col, '')\n            row_list.append(str(value) if value is not None else '')\n        \n        try:\n            # Update entire row\n            range_name = f'A{row_number}:{self._get_column_letter(len(self.columns))}{row_number}'\n            self.sheet.update(range_name, [row_list], value_input_option='RAW')\n            print(f\"✓ Updated complete results for experiment {experiment_id}\")\n        except Exception as e:\n            print(f\"✗ Failed to update experiment row: {e}\")\n            traceback.print_exc()\n    \n    def _prepare_row_data(self, config: ExperimentConfig, results: Dict[str, Any]) -> Dict[str, Any]:\n        config_hash = config.get_hash()\n        \n        feature_group_config = Config.FEATURE_GROUP_CONFIG\n        \n        # Categorize features\n        market_features = [f for f in config.feature_list if f in feature_group_config.market_features]\n        microstructure_features = [f for f in config.feature_list if f in feature_group_config.microstructure_features]\n        core_proprietary = [f for f in config.feature_list if f in feature_group_config.core_proprietary_features]\n        additional_x = config.additional_features\n        \n        # Calculate improvement\n        improvement = 0\n        if self.baseline_score and results.get('mean_val_score'):\n            improvement = results['mean_val_score'] - self.baseline_score\n        \n        # Calculate total parameters\n        total_params = 0\n        if config.hidden_dims:\n            prev_dim = len(config.feature_list)\n            for hidden_dim in config.hidden_dims:\n                total_params += prev_dim * hidden_dim + hidden_dim\n                prev_dim = hidden_dim\n        \n        # Check for overfitting\n        overfitting_flag = False\n        if results.get('train_val_gap', 0) > Config.MAX_TRAIN_VAL_GAP:\n            overfitting_flag = True\n        \n        # Convert numpy types for JSON serialization\n        fold_scores = to_python_type(results.get('fold_scores', []))\n        fold_train_scores = to_python_type(results.get('fold_train_scores', []))\n        \n        row_data = {\n            'experiment_id': results.get('experiment_id', ''),\n            'timestamp_start': results.get('timestamp_start', ''),\n            'timestamp_end': results.get('timestamp_end', datetime.now().isoformat()),\n            'status': results.get('status', 'completed'),\n            'config_hash': config_hash,\n            'baseline_similarity_score': results.get('baseline_similarity', 0),\n            \n            'core_features_list': json.dumps(core_proprietary),\n            'n_core_features': len(core_proprietary),\n            'market_features_list': json.dumps(market_features),\n            'n_market_features': len(market_features),\n            'microstructure_features_list': json.dumps(microstructure_features),\n            'n_microstructure_features': len(microstructure_features),\n            'additional_x_features_list': json.dumps(additional_x),\n            'n_additional_x_features': len(additional_x),\n            'total_features': len(config.feature_list),\n            \n            'core_features_transform': 'standard',\n            'market_features_transform': 'standard',\n            'x_features_transform': json.dumps(config.feature_transforms),\n            'global_transforms': json.dumps(config.feature_transforms),\n            'infinity_strategy': config.infinity_strategy,\n            \n            'use_interactions': config.use_interactions,\n            'interaction_pairs': json.dumps(results.get('interaction_pairs', [])),\n            'n_interactions': len(results.get('interaction_pairs', [])),\n            'interaction_method': config.interaction_method,\n            'interaction_features_source': json.dumps(config.interaction_features),\n            \n            'mean_train_score': round(float(results.get('mean_train_score', 0)), 4),\n            'mean_val_score': round(float(results.get('mean_val_score', 0)), 4),\n            'std_train_score': round(float(results.get('std_train_score', 0)), 4),\n            'std_val_score': round(float(results.get('std_val_score', 0)), 4),\n            'train_val_gap': round(float(results.get('train_val_gap', 0)), 4),\n            'best_fold_score': round(float(results.get('best_fold_score', 0)), 4),\n            'worst_fold_score': round(float(results.get('worst_fold_score', 0)), 4),\n            'improvement_over_baseline': round(improvement, 4),\n            'overfitting_flag': overfitting_flag,\n            \n            'architecture_name': config.architecture_name,\n            'architecture_category': config.architecture_type,\n            'hidden_dims': json.dumps(config.hidden_dims),\n            'total_parameters': total_params,\n            'depth': len(config.hidden_dims) if config.hidden_dims else 0,\n            'width': max(config.hidden_dims) if config.hidden_dims else 0,\n            \n            'activation': config.activation,\n            'dropout_rates_dict': json.dumps(config.dropout_rates_dict),\n            'optimizer': config.optimizer,\n            'learning_rate': config.learning_rate,\n            'weight_decay': config.weight_decay,\n            'noise_levels_dict': json.dumps(config.noise_levels_dict),\n            'batch_size': Config.BATCH_SIZE,\n            'input_noise': config.input_noise,\n            \n            'use_clustering': config.use_clustering,\n            'n_clusters': config.n_clusters,\n            'use_pca': config.use_pca,\n            'pca_components': config.pca_components,\n            'use_binning': config.use_binning,\n            'binning_details': json.dumps({\n                'method': config.binning_method,\n                'features': config.binning_features,\n                'n_bins': config.n_bins\n            }),\n            \n            'label_transform': config.label_transform,\n            'cv_strategy': config.cv_strategy,\n            'n_folds': results.get('n_folds', Config.N_FOLDS),\n            'n_ensemble': config.n_ensemble,\n            'gradient_clip': config.gradient_clip,\n            'use_mixup': config.use_mixup,\n            'use_cutmix': config.use_cutmix,\n            'use_batch_norm': config.use_batch_norm,\n            'use_layer_norm': config.use_layer_norm,\n            'use_residual': config.use_residual,\n            \n            'training_time_minutes': round(float(results.get('training_time_minutes', 0)), 2),\n            'memory_peak_gb': round(float(results.get('memory_peak_gb', 0)), 2),\n            'submission_file': results.get('submission_file', ''),\n            'error_message': results.get('error_message', ''),\n            \n            'fold_scores_json': json.dumps(fold_scores),\n            'fold_train_scores_json': json.dumps(fold_train_scores),\n            'config_full_json': json.dumps(config.__dict__, default=str)\n        }\n        \n        return row_data\n    \n    def _get_column_letter(self, n):\n        \"\"\"Convert column number to letter (1=A, 2=B, etc.)\"\"\"\n        string = \"\"\n        while n > 0:\n            n, remainder = divmod(n - 1, 26)\n            string = chr(65 + remainder) + string\n        return string\n    \n    def set_baseline_score(self, score: float):\n        self.baseline_score = score\n        print(f\"Set baseline score: {score:.4f}\")\n\nclass BaselineMLP(nn.Module):\n    def __init__(self, input_dim: int, \n                 layers: List[int] = [256, 64, 1],\n                 dropout_rate: float = 0.7,\n                 activation: str = 'relu',\n                 noise_factor: float = 0.01):\n        super().__init__()\n        \n        self.noise_factor = noise_factor\n        self.layers_dims = [input_dim] + layers\n        \n        self.linears = nn.ModuleList()\n        self.dropouts = nn.ModuleList()\n        \n        for i in range(len(self.layers_dims) - 1):\n            self.linears.append(\n                nn.Linear(self.layers_dims[i], self.layers_dims[i + 1])\n            )\n            # Higher dropout for later layers\n            if i < len(self.layers_dims) - 2:\n                dropout = min(dropout_rate + i * 0.1, 0.9)\n                self.dropouts.append(nn.Dropout(dropout))\n        \n        self.activation = self._get_activation(activation)\n    \n    def _get_activation(self, name: str):\n        activations = {\n            'relu': nn.ReLU(),\n            'tanh': nn.Tanh(),\n            'sigmoid': nn.Sigmoid(),\n            'gelu': nn.GELU(),\n            'silu': nn.SiLU(),\n            'leaky_relu': nn.LeakyReLU(0.1)\n        }\n        return activations.get(name.lower(), nn.ReLU())\n    \n    def forward(self, x):\n        if self.training and self.noise_factor > 0:\n            noise = torch.randn_like(x) * self.noise_factor\n            x = x + noise\n        \n        for i in range(len(self.linears) - 1):\n            x = self.linears[i](x)\n            x = self.activation(x)\n            if i < len(self.dropouts):\n                x = self.dropouts[i](x)\n        \n        x = self.linears[-1](x)\n        return x.squeeze()\n\nclass CryptoModel(nn.Module):\n    def __init__(self, input_dim: int, config: ExperimentConfig, feature_indices: Dict[str, List[int]] = None):\n        super().__init__()\n        self.config = config\n        self.input_noise = config.input_noise\n        \n        if config.architecture_name == 'baseline_mlp':\n            self.model = BaselineMLP(\n                input_dim, \n                layers=config.hidden_dims,\n                dropout_rate=config.dropout_rate,\n                activation=config.activation,\n                noise_factor=config.input_noise\n            )\n        else:\n            # Generic MLP for other architectures\n            layers = []\n            prev_dim = input_dim\n            \n            for i, hidden_dim in enumerate(config.hidden_dims[:-1]):\n                layers.append(nn.Linear(prev_dim, hidden_dim))\n                \n                # Activation\n                if config.activation == 'relu':\n                    layers.append(nn.ReLU())\n                elif config.activation == 'tanh':\n                    layers.append(nn.Tanh())\n                elif config.activation == 'leaky_relu':\n                    layers.append(nn.LeakyReLU(0.1))\n                else:\n                    layers.append(nn.ReLU())\n                \n                # Increasing dropout\n                dropout_rate = min(config.dropout_rate + i * 0.05, 0.9)\n                layers.append(nn.Dropout(dropout_rate))\n                \n                prev_dim = hidden_dim\n            \n            # Output layer\n            layers.append(nn.Linear(prev_dim, config.hidden_dims[-1]))\n            \n            self.model = nn.Sequential(*layers)\n        \n        self.apply(self._init_weights)\n    \n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            # Conservative initialization\n            nn.init.xavier_normal_(m.weight, gain=0.5)\n            if m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n    \n    def forward(self, x):\n        if self.training and self.input_noise > 0:\n            noise = torch.randn_like(x) * self.input_noise\n            x = x + noise\n        \n        return self.model(x).squeeze()\n\nclass FeatureEngineer:\n    def __init__(self, config: ExperimentConfig):\n        self.config = config\n        self.fitted = False\n        self.transformers = {}\n        self.feature_stats = {}\n        \n    def fit(self, X: pd.DataFrame, y: np.ndarray):\n        # Calculate feature statistics for handling infinities\n        for col in X.columns:\n            values = X[col].values\n            finite_mask = np.isfinite(values)\n            if finite_mask.sum() > 0:\n                finite_values = values[finite_mask]\n                self.feature_stats[col] = {\n                    'median': np.median(finite_values),\n                    'mean': np.mean(finite_values),\n                    'std': np.std(finite_values),\n                    'p5': np.percentile(finite_values, 5),\n                    'p95': np.percentile(finite_values, 95)\n                }\n        \n        X_clean = self._handle_infinity(X.copy())\n        \n        # Fit transformers\n        if 'standard' in self.config.feature_transforms:\n            self.transformers['standard'] = StandardScaler()\n            self.transformers['standard'].fit(X_clean)\n        \n        if 'robust' in self.config.feature_transforms:\n            self.transformers['robust'] = RobustScaler()\n            self.transformers['robust'].fit(X_clean)\n        \n        if 'quantile' in self.config.feature_transforms:\n            self.transformers['quantile'] = QuantileTransformer(\n                n_quantiles=min(1000, len(X_clean)),\n                output_distribution='normal'\n            )\n            self.transformers['quantile'].fit(X_clean)\n        \n        self.fitted = True\n    \n    def transform(self, X: pd.DataFrame) -> np.ndarray:\n        if not self.fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        X_clean = self._handle_infinity(X.copy())\n        \n        all_features = []\n        \n        # Apply transformations\n        if 'standard' in self.config.feature_transforms:\n            all_features.append(self.transformers['standard'].transform(X_clean))\n        \n        if 'robust' in self.config.feature_transforms and 'robust' in self.transformers:\n            all_features.append(self.transformers['robust'].transform(X_clean))\n        \n        if 'rank' in self.config.feature_transforms:\n            rank_features = np.apply_along_axis(\n                lambda x: rankdata(x, method='average') / (len(x) + 1), 0, X_clean.values\n            )\n            all_features.append(rank_features)\n        \n        if 'quantile' in self.config.feature_transforms and 'quantile' in self.transformers:\n            all_features.append(self.transformers['quantile'].transform(X_clean))\n        \n        if len(all_features) == 0:\n            return X_clean.values\n        \n        return np.hstack(all_features)\n    \n    def _handle_infinity(self, X: pd.DataFrame) -> pd.DataFrame:\n        for col in X.columns:\n            if col not in self.feature_stats:\n                # Handle new columns\n                finite_mask = np.isfinite(X[col])\n                if finite_mask.sum() > 0:\n                    median_val = np.median(X[col][finite_mask])\n                    X.loc[~finite_mask, col] = median_val\n                else:\n                    X[col] = 0\n                continue\n            \n            stats = self.feature_stats[col]\n            \n            if self.config.infinity_strategy == 'median':\n                X.loc[~np.isfinite(X[col]), col] = stats['median']\n            elif self.config.infinity_strategy == 'percentile':\n                X.loc[X[col] == np.inf, col] = stats['p95']\n                X.loc[X[col] == -np.inf, col] = stats['p5']\n                X.loc[np.isnan(X[col]), col] = stats['median']\n            elif self.config.infinity_strategy == 'zero':\n                X.loc[~np.isfinite(X[col]), col] = 0\n        \n        return X\n\nclass ModelTrainer:\n    def __init__(self, config: ExperimentConfig, memory_manager: MemoryManager):\n        self.config = config\n        self.memory_manager = memory_manager\n        \n    def train_with_cv(self, X_train: np.ndarray, y_train: np.ndarray, \n                      feature_indices: Dict[str, List[int]] = None) -> Dict[str, Any]:\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        \n        fold_train_scores = []\n        fold_val_scores = []\n        models = []\n        \n        # Create CV splits based on strategy\n        if self.config.cv_strategy == 'kfold':\n            cv = KFold(n_splits=Config.N_FOLDS, shuffle=True, random_state=42)\n            splits = list(cv.split(X_train))\n            \n        elif self.config.cv_strategy == 'stratified':\n            # Create bins for stratification\n            y_bins = pd.qcut(y_train, q=10, labels=False, duplicates='drop')\n            cv = StratifiedKFold(n_splits=Config.N_FOLDS, shuffle=True, random_state=42)\n            splits = list(cv.split(X_train, y_bins))\n            \n        elif self.config.cv_strategy == 'timeseries_gap':\n            # Time series with gap to prevent leakage\n            n_samples = len(X_train)\n            gap_size = n_samples // 50  # 2% gap\n            splits = []\n            \n            for i in range(Config.N_FOLDS):\n                # Create train/val split with gap\n                val_size = (n_samples - gap_size * Config.N_FOLDS) // Config.N_FOLDS\n                val_start = i * (val_size + gap_size) + gap_size\n                val_end = val_start + val_size\n                \n                train_idx = np.concatenate([\n                    np.arange(0, val_start - gap_size),\n                    np.arange(val_end + gap_size, n_samples)\n                ])\n                val_idx = np.arange(val_start, val_end)\n                \n                splits.append((train_idx, val_idx))\n        \n        print(f\"\\nTraining with {len(splits)} folds using {self.config.cv_strategy} strategy\")\n        \n        for fold_idx, (train_idx, val_idx) in enumerate(splits):\n            print(f\"\\nFold {fold_idx + 1}/{len(splits)}\")\n            \n            X_fold_train = X_train[train_idx]\n            y_fold_train = y_train[train_idx]\n            X_fold_val = X_train[val_idx]\n            y_fold_val = y_train[val_idx]\n            \n            train_score, val_score, model = self._train_single_model(\n                X_fold_train, y_fold_train, X_fold_val, y_fold_val,\n                feature_indices=feature_indices,\n                seed=42 + fold_idx\n            )\n            \n            fold_train_scores.append(train_score)\n            fold_val_scores.append(val_score)\n            models.append(model)\n            \n            print(f\"  Train: {train_score:.4f}, Val: {val_score:.4f}, Gap: {train_score - val_score:.4f}\")\n            \n            # Early stopping if overfitting\n            if train_score - val_score > Config.MAX_TRAIN_VAL_GAP * 2:\n                print(\"  ⚠️  Severe overfitting detected, stopping CV early\")\n                break\n            \n            self.memory_manager.clear_memory()\n        \n        results = {\n            'fold_train_scores': fold_train_scores,\n            'fold_scores': fold_val_scores,\n            'mean_train_score': np.mean(fold_train_scores),\n            'mean_val_score': np.mean(fold_val_scores),\n            'std_train_score': np.std(fold_train_scores),\n            'std_val_score': np.std(fold_val_scores),\n            'train_val_gap': np.mean(fold_train_scores) - np.mean(fold_val_scores),\n            'best_fold_score': np.max(fold_val_scores) if fold_val_scores else 0,\n            'worst_fold_score': np.min(fold_val_scores) if fold_val_scores else 0,\n            'models': models,\n            'n_folds': len(fold_val_scores),\n            'overfitting_flag': (np.mean(fold_train_scores) - np.mean(fold_val_scores)) > Config.MAX_TRAIN_VAL_GAP\n        }\n        \n        return results\n    \n    def _train_single_model(self, X_train: np.ndarray, y_train: np.ndarray,\n                           X_val: np.ndarray, y_val: np.ndarray,\n                           feature_indices: Dict[str, List[int]] = None,\n                           seed: int = 42) -> Tuple[float, float, nn.Module]:\n        torch.manual_seed(seed)\n        np.random.seed(seed)\n        \n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        \n        # Transform labels if needed\n        y_train_transformed = self._transform_labels(y_train)\n        y_val_transformed = self._transform_labels(y_val)\n        \n        # Create model\n        model = CryptoModel(X_train.shape[1], self.config, feature_indices).to(device)\n        \n        # Optimizer with higher weight decay\n        optimizer = optim.Adam(\n            model.parameters(), \n            lr=self.config.learning_rate,\n            weight_decay=self.config.weight_decay\n        )\n        \n        # Aggressive scheduler\n        scheduler = optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer, mode='max', patience=2, factor=0.5, min_lr=1e-6\n        )\n        \n        # Create data loaders\n        train_dataset = TensorDataset(\n            torch.FloatTensor(X_train),\n            torch.FloatTensor(y_train_transformed)\n        )\n        val_dataset = TensorDataset(\n            torch.FloatTensor(X_val),\n            torch.FloatTensor(y_val_transformed)\n        )\n        \n        train_loader = DataLoader(train_dataset, batch_size=Config.BATCH_SIZE, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=Config.BATCH_SIZE * 2, shuffle=False)\n        \n        # Loss function with regularization\n        criterion = nn.HuberLoss(delta=1.0)\n        \n        best_val_score = -np.inf\n        patience_counter = 0\n        best_model_state = None\n        train_scores = []\n        val_scores = []\n        \n        for epoch in range(Config.MAX_EPOCHS):\n            # Training\n            model.train()\n            train_losses = []\n            \n            for batch_X, batch_y in train_loader:\n                batch_X, batch_y = batch_X.to(device), batch_y.to(device)\n                \n                optimizer.zero_grad()\n                outputs = model(batch_X)\n                loss = criterion(outputs, batch_y)\n                \n                # Add L2 penalty\n                l2_penalty = sum(p.pow(2).sum() for p in model.parameters())\n                loss = loss + self.config.weight_decay * l2_penalty\n                \n                loss.backward()\n                \n                # Gradient clipping\n                torch.nn.utils.clip_grad_norm_(model.parameters(), self.config.gradient_clip)\n                \n                optimizer.step()\n                train_losses.append(loss.item())\n            \n            # Validation\n            model.eval()\n            val_preds = []\n            val_targets = []\n            \n            with torch.no_grad():\n                for batch_X, batch_y in val_loader:\n                    batch_X = batch_X.to(device)\n                    outputs = model(batch_X)\n                    val_preds.extend(outputs.cpu().numpy())\n                    val_targets.extend(batch_y.numpy())\n            \n            val_preds = np.array(val_preds)\n            val_targets = np.array(val_targets)\n            \n            # Calculate scores\n            val_score = pearsonr(val_targets, val_preds)[0]\n            val_score = 0 if np.isnan(val_score) else val_score\n            \n            # Calculate train score for monitoring\n            model.eval()\n            with torch.no_grad():\n                train_preds = []\n                for batch_X, _ in train_loader:\n                    batch_X = batch_X.to(device)\n                    outputs = model(batch_X)\n                    train_preds.extend(outputs.cpu().numpy())\n                train_score = pearsonr(y_train_transformed, train_preds)[0]\n                train_score = 0 if np.isnan(train_score) else train_score\n            \n            train_scores.append(train_score)\n            val_scores.append(val_score)\n            \n            # Check for overfitting\n            if train_score - val_score > Config.MAX_TRAIN_VAL_GAP * 1.5:\n                print(f\"    Epoch {epoch}: Overfitting detected (gap: {train_score - val_score:.4f})\")\n                break\n            \n            scheduler.step(val_score)\n            \n            if val_score > best_val_score:\n                best_val_score = val_score\n                patience_counter = 0\n                best_model_state = model.state_dict().copy()\n            else:\n                patience_counter += 1\n                if patience_counter >= Config.EARLY_STOPPING_PATIENCE:\n                    break\n        \n        # Load best model\n        if best_model_state is not None:\n            model.load_state_dict(best_model_state)\n        \n        # Final evaluation\n        model.eval()\n        with torch.no_grad():\n            train_preds = model(torch.FloatTensor(X_train).to(device)).cpu().numpy()\n            train_score = pearsonr(y_train_transformed, train_preds)[0]\n            train_score = 0 if np.isnan(train_score) else train_score\n        \n        return train_score, best_val_score, model\n    \n    def _transform_labels(self, y: np.ndarray) -> np.ndarray:\n        if self.config.label_transform == 'none':\n            return y\n        elif self.config.label_transform == 'rank':\n            return rankdata(y) / (len(y) + 1)\n        elif self.config.label_transform == 'quantile':\n            transformer = QuantileTransformer(n_quantiles=min(1000, len(y)), output_distribution='normal')\n            return transformer.fit_transform(y.reshape(-1, 1)).ravel()\n        return y\n\nclass ExperimentRunner:\n    def __init__(self, memory_manager: MemoryManager, sheets_tracker: Optional[EnhancedSheetsTracker] = None):\n        self.memory_manager = memory_manager\n        self.sheets_tracker = sheets_tracker\n        self.feature_group_config = Config.FEATURE_GROUP_CONFIG\n        \n    def run_experiment(self, train_df: pd.DataFrame, test_df: pd.DataFrame,\n                      config: ExperimentConfig, \n                      config_selector: Optional['IntelligentConfigSelector'] = None) -> Dict[str, Any]:\n        start_time = time.time()\n        experiment_id = f\"exp_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{random.randint(1000, 9999)}\"\n        \n        # Check if already tested\n        if self.sheets_tracker and self.sheets_tracker.config_exists(config):\n            print(f\"Configuration already tested: {config.get_hash()}\")\n            return {'success': False, 'reason': 'already_tested'}\n        \n        print(f\"\\n{'='*60}\")\n        print(f\"Starting experiment {experiment_id}\")\n        print(f\"Config: {config.get_description()}\")\n        print(f\"{'='*60}\")\n        \n        # Log start to spreadsheet\n        if self.sheets_tracker:\n            row_num = self.sheets_tracker.log_experiment_start(experiment_id, config)\n            if row_num == -1:\n                print(\"Failed to log experiment start\")\n        \n        try:\n            # Update status to running\n            if self.sheets_tracker:\n                self.sheets_tracker.update_experiment_status(experiment_id, \"Running\")\n            \n            # Check memory\n            if not self.memory_manager.check_memory_available(Config.MIN_MEMORY_GB):\n                raise MemoryError(\"Insufficient memory available\")\n            \n            # Prepare data\n            X_train = train_df[config.feature_list]\n            y_train = train_df['label'].values\n            X_test = test_df[config.feature_list]\n            \n            # USE FULL DATASET - NO SAMPLING!\n            print(f\"Training on FULL dataset: {len(X_train)} samples\")\n            X_train_full = X_train\n            y_train_full = y_train\n            \n            # Feature engineering\n            engineer = FeatureEngineer(config)\n            engineer.fit(X_train_full, y_train_full)\n            \n            X_train_transformed = engineer.transform(X_train_full)\n            X_test_transformed = engineer.transform(X_test)\n            \n            print(f\"Feature dimensions: {X_train_transformed.shape[1]}\")\n            \n            # Get feature indices for hierarchical models\n            feature_indices = self.feature_group_config.get_feature_indices(config.feature_list)\n            \n            # Train with CV\n            trainer = ModelTrainer(config, self.memory_manager)\n            cv_results = trainer.train_with_cv(X_train_transformed, y_train_full, feature_indices)\n            \n            print(f\"\\nCV Results:\")\n            print(f\"  Mean train: {cv_results['mean_train_score']:.4f} ± {cv_results['std_train_score']:.4f}\")\n            print(f\"  Mean val: {cv_results['mean_val_score']:.4f} ± {cv_results['std_val_score']:.4f}\")\n            print(f\"  Train-val gap: {cv_results['train_val_gap']:.4f}\")\n            print(f\"  Overfitting: {'Yes' if cv_results['overfitting_flag'] else 'No'}\")\n            \n            # Update status\n            if self.sheets_tracker:\n                self.sheets_tracker.update_experiment_status(\n                    experiment_id, \n                    \"Generating predictions\",\n                    cv_results\n                )\n            \n            # Generate predictions\n            device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n            all_preds = []\n            \n            for model in cv_results['models']:\n                model.eval()\n                with torch.no_grad():\n                    preds = model(torch.FloatTensor(X_test_transformed).to(device)).cpu().numpy()\n                    all_preds.append(preds)\n            \n            test_preds = np.mean(all_preds, axis=0)\n            \n            # Save submission\n            submission_file = f\"submission_{config.get_hash()}.csv\"\n            submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n            submission_df['prediction'] = test_preds\n            submission_df.to_csv(submission_file, index=False)\n            print(f\"Saved predictions to {submission_file}\")\n            \n            # Calculate baseline similarity\n            baseline_similarity = 0\n            if config_selector:\n                baseline_similarity = config_selector.calculate_similarity_to_baseline(config)\n            \n            # Prepare results\n            results = {\n                'success': True,\n                'experiment_id': experiment_id,\n                'timestamp_start': datetime.fromtimestamp(start_time).isoformat(),\n                'timestamp_end': datetime.now().isoformat(),\n                'mean_train_score': cv_results['mean_train_score'],\n                'mean_val_score': cv_results['mean_val_score'],\n                'std_train_score': cv_results['std_train_score'],\n                'std_val_score': cv_results['std_val_score'],\n                'train_val_gap': cv_results['train_val_gap'],\n                'best_fold_score': cv_results['best_fold_score'],\n                'worst_fold_score': cv_results['worst_fold_score'],\n                'fold_scores': cv_results['fold_scores'],\n                'fold_train_scores': cv_results['fold_train_scores'],\n                'n_folds': cv_results['n_folds'],\n                'training_time_minutes': (time.time() - start_time) / 60,\n                'memory_peak_gb': self.memory_manager.get_memory_info()['cpu_used_gb'],\n                'submission_file': submission_file,\n                'baseline_similarity': baseline_similarity,\n                'overfitting_flag': cv_results['overfitting_flag'],\n                'status': 'Completed'\n            }\n            \n            # Update importance scores\n            if config_selector:\n                config_selector.update_importance_scores(config, cv_results['mean_val_score'])\n            \n            # Log complete results\n            if self.sheets_tracker:\n                self.sheets_tracker.update_experiment_status(experiment_id, \"Completed\", results)\n                self.sheets_tracker.log_experiment_complete(experiment_id, config, results)\n            \n            # Clean up\n            del cv_results['models']\n            self.memory_manager.clear_memory()\n            \n            print(f\"✓ Experiment completed in {results['training_time_minutes']:.1f} minutes\")\n            \n            return results\n            \n        except Exception as e:\n            error_msg = f\"{type(e).__name__}: {str(e)}\"\n            print(f\"✗ Experiment failed: {error_msg}\")\n            traceback.print_exc()\n            \n            results = {\n                'success': False,\n                'experiment_id': experiment_id,\n                'timestamp_start': datetime.fromtimestamp(start_time).isoformat(),\n                'timestamp_end': datetime.now().isoformat(),\n                'status': 'Error',\n                'error_message': error_msg[:500],\n                'training_time_minutes': (time.time() - start_time) / 60\n            }\n            \n            # Log error\n            if self.sheets_tracker:\n                self.sheets_tracker.update_experiment_status(experiment_id, \"Error\", results)\n                self.sheets_tracker.log_experiment_complete(experiment_id, config, results)\n            \n            return results\n\nclass IntelligentConfigSelector:\n    def __init__(self, baseline_config: ExperimentConfig):\n        self.baseline_config = baseline_config\n        self.performance_history = []\n        self.feature_importance = defaultdict(float)\n        self.param_importance = defaultdict(float)\n        \n    def calculate_similarity_to_baseline(self, config: ExperimentConfig) -> float:\n        similarity = 0.0\n        weights = {\n            'architecture': 0.3,\n            'hyperparams': 0.4,\n            'features': 0.3\n        }\n        \n        # Architecture similarity\n        if config.hidden_dims == self.baseline_config.hidden_dims:\n            similarity += weights['architecture']\n        \n        # Hyperparameter similarity\n        hyperparam_matches = 0\n        hyperparams = ['activation', 'dropout_rate', 'learning_rate', 'optimizer', 'weight_decay']\n        for param in hyperparams:\n            if getattr(config, param) == getattr(self.baseline_config, param):\n                hyperparam_matches += 1\n        similarity += weights['hyperparams'] * (hyperparam_matches / len(hyperparams))\n        \n        # Feature similarity\n        if len(config.additional_features) == 0:\n            similarity += weights['features']\n        \n        return similarity\n    \n    def update_importance_scores(self, config: ExperimentConfig, score: float):\n        # Update feature importance\n        for feature in config.additional_features:\n            self.feature_importance[feature] = (\n                0.9 * self.feature_importance[feature] + 0.1 * score\n            )\n        \n        # Update parameter importance\n        param_key = f\"{config.architecture_name}_{config.activation}_{config.dropout_rate}\"\n        self.param_importance[param_key] = (\n            0.9 * self.param_importance[param_key] + 0.1 * score\n        )\n        \n        # Store in history\n        self.performance_history.append({\n            'config': config,\n            'score': score,\n            'timestamp': datetime.now()\n        })\n    \n    def prioritize_configs(self, configs: List[ExperimentConfig], \n                          n_select: int = 30) -> List[ExperimentConfig]:\n        if len(self.performance_history) < 5:\n            # Random selection initially\n            return random.sample(configs, min(n_select, len(configs)))\n        \n        config_scores = []\n        \n        for config in configs:\n            # Calculate expected score\n            baseline_similarity = self.calculate_similarity_to_baseline(config)\n            \n            feature_score = 0\n            if config.additional_features:\n                feature_scores = [\n                    self.feature_importance.get(f, 0.01) \n                    for f in config.additional_features\n                ]\n                feature_score = np.mean(feature_scores)\n            \n            # Penalize overly complex models\n            complexity_penalty = len(config.additional_features) * 0.02\n            \n            final_score = (\n                0.4 * baseline_similarity + \n                0.6 * feature_score - \n                complexity_penalty\n            )\n            \n            config_scores.append((config, final_score))\n        \n        # Sort by score\n        config_scores.sort(key=lambda x: x[1], reverse=True)\n        \n        # Select top configs with some randomness\n        selected = []\n        for i, (config, score) in enumerate(config_scores):\n            if i < n_select * 0.7:  # Top 70%\n                selected.append(config)\n            elif i < n_select and random.random() < 0.5:  # Random 30%\n                selected.append(config)\n            \n            if len(selected) >= n_select:\n                break\n        \n        return selected\n\nclass ConfigGenerator:\n    def __init__(self, core_features: List[str], all_features: List[str], \n                 baseline_config: ExperimentConfig):\n        self.core_features = core_features\n        self.additional_features = [f for f in all_features if f not in core_features and f.startswith('X')]\n        self.additional_features.sort()\n        self.baseline_config = baseline_config\n        \n    def generate_configs_for_features(self, feature_combination: List[str]) -> List[ExperimentConfig]:\n        configs = []\n        feature_list = self.core_features + list(feature_combination)\n        \n        # Baseline configuration\n        if not feature_combination:\n            configs.append(self.baseline_config)\n        \n        # Generate variations\n        param_grid = {\n            'infinity_strategy': Config.INFINITY_STRATEGIES[:2],\n            'feature_transforms': Config.FEATURE_TRANSFORMS[:3],\n            'label_transform': Config.LABEL_TRANSFORMS[:2],\n            'architecture': Config.ARCHITECTURE_VARIANTS[:3],\n            'activation': Config.ACTIVATIONS[:2],\n            'dropout_rate': Config.DROPOUT_RATES[:2],\n            'learning_rate': Config.LEARNING_RATES[:2],\n            'weight_decay': Config.WEIGHT_DECAYS[:2],\n            'cv_strategy': ['kfold', 'timeseries_gap']\n        }\n        \n        # Create configurations\n        for params in self._generate_param_combinations(param_grid, max_configs=Config.MAX_CONFIGS_PER_FEATURE):\n            config = ExperimentConfig(\n                feature_list=feature_list,\n                additional_features=list(feature_combination),\n                infinity_strategy=params['infinity_strategy'],\n                feature_transforms=params['feature_transforms'],\n                label_transform=params['label_transform'],\n                architecture_name=params['architecture']['name'],\n                architecture_type=params['architecture']['type'],\n                hidden_dims=params['architecture']['hidden_dims'],\n                activation=params['activation'],\n                dropout_rate=params['dropout_rate'],\n                learning_rate=params['learning_rate'],\n                weight_decay=params['weight_decay'],\n                cv_strategy=params['cv_strategy'],\n                optimizer='adam',\n                gradient_clip=1.0,\n                input_noise=0.02,\n                dropout_rates_dict=Config.FEATURE_GROUP_CONFIG.dropout_rates,\n                noise_levels_dict=Config.FEATURE_GROUP_CONFIG.noise_levels\n            )\n            configs.append(config)\n        \n        return configs\n    \n    def _generate_param_combinations(self, param_grid: Dict, max_configs: int):\n        \"\"\"Generate parameter combinations with intelligent sampling\"\"\"\n        # Get all keys and values\n        keys = list(param_grid.keys())\n        values = [param_grid[key] for key in keys]\n        \n        # Calculate total combinations\n        total = 1\n        for v in values:\n            total *= len(v)\n        \n        # If total is reasonable, generate all\n        if total <= max_configs:\n            for combination in itertools.product(*values):\n                yield dict(zip(keys, combination))\n        else:\n            # Random sampling with proper handling of all data types\n            seen = set()\n            attempts = 0\n            max_attempts = max_configs * 10\n            \n            while len(seen) < max_configs and attempts < max_attempts:\n                attempts += 1\n                \n                # Create combination\n                combination_values = []\n                for v in values:\n                    choice = random.choice(v)\n                    combination_values.append(choice)\n                \n                # Create hashable key using the make_hashable function\n                hashable_key = make_hashable(combination_values)\n                \n                if hashable_key not in seen:\n                    seen.add(hashable_key)\n                    # Convert back to original format\n                    yield dict(zip(keys, combination_values))\n\nclass EnhancedCryptoPipeline:\n    def __init__(self):\n        self.start_time = time.time()\n        self.memory_manager = MemoryManager()\n        self.sheets_tracker = None\n        self.experiment_runner = None\n        self.baseline_config = create_baseline_config()\n        self.baseline_score = None\n        \n        # Initialize tracking\n        if GSPREAD_AVAILABLE and os.path.exists(Config.CREDENTIALS_FILE):\n            try:\n                self.sheets_tracker = EnhancedSheetsTracker(\n                    Config.CREDENTIALS_FILE,\n                    Config.SPREADSHEET_URL\n                )\n                self.experiment_runner = ExperimentRunner(\n                    self.memory_manager, self.sheets_tracker\n                )\n                print(\"✓ Google Sheets tracking initialized\")\n            except Exception as e:\n                print(f\"✗ Failed to initialize sheets tracker: {e}\")\n                self.experiment_runner = ExperimentRunner(self.memory_manager)\n        else:\n            print(\"Running without Google Sheets tracking\")\n            self.experiment_runner = ExperimentRunner(self.memory_manager)\n    \n    def run(self):\n        print(\"=\"*80)\n        print(\"CRYPTO PREDICTION PIPELINE - FULL DATASET VERSION\")\n        print(\"=\"*80)\n        \n        # Load data\n        print(\"\\nLoading data...\")\n        train_df = pd.read_parquet(Config.TRAIN_PATH)\n        test_df = pd.read_parquet(Config.TEST_PATH)\n        \n        # Create microstructure features\n        print(\"Creating microstructure features...\")\n        train_df = create_advanced_microstructure_features(train_df)\n        test_df = create_advanced_microstructure_features(test_df)\n        \n        print(f\"Train shape: {train_df.shape}\")\n        print(f\"Test shape: {test_df.shape}\")\n        print(f\"Using FULL training dataset: {len(train_df)} samples\")\n        \n        # Check core features\n        missing_features = [f for f in Config.CORE_FEATURES if f not in train_df.columns]\n        if missing_features:\n            print(f\"Warning: Missing core features: {missing_features}\")\n            Config.CORE_FEATURES = [f for f in Config.CORE_FEATURES if f in train_df.columns]\n        \n        print(f\"Using {len(Config.CORE_FEATURES)} core features\")\n        \n        # Get all features\n        all_features = [col for col in train_df.columns if col not in ['label', 'timestamp']]\n        \n        # Initialize generators\n        config_generator = ConfigGenerator(Config.CORE_FEATURES, all_features, self.baseline_config)\n        config_selector = IntelligentConfigSelector(self.baseline_config)\n        \n        print(f\"Found {len(config_generator.additional_features)} additional X features\")\n        \n        # Tracking variables\n        n_completed = 0\n        n_errors = 0\n        n_skipped = 0\n        best_score = -np.inf\n        best_config = None\n        \n        # Phase 0: Baseline\n        print(\"\\n\" + \"=\"*60)\n        print(\"PHASE 0: Testing baseline model\")\n        print(\"=\"*60)\n        \n        baseline_result = self.experiment_runner.run_experiment(\n            train_df, test_df, self.baseline_config, config_selector\n        )\n        \n        if baseline_result['success']:\n            self.baseline_score = baseline_result['mean_val_score']\n            best_score = self.baseline_score\n            best_config = self.baseline_config\n            n_completed += 1\n            \n            if self.sheets_tracker:\n                self.sheets_tracker.set_baseline_score(self.baseline_score)\n            \n            print(f\"\\n✓ Baseline score: {self.baseline_score:.4f}\")\n        else:\n            print(\"\\n✗ Failed to run baseline model\")\n            if baseline_result.get('reason') != 'already_tested':\n                return\n            else:\n                # Set a default baseline score if already tested\n                self.baseline_score = 0.01\n                print(\"Using default baseline score: 0.01\")\n        \n        # Phase 1: Individual features\n        print(\"\\n\" + \"=\"*60)\n        print(\"PHASE 1: Testing individual additional features\")\n        print(\"=\"*60)\n        \n        feature_scores = {}\n        \n        # Test top features only\n        features_to_test = config_generator.additional_features[:30]\n        \n        for feature_idx, feature in enumerate(features_to_test):\n            print(f\"\\n--- Testing feature {feature_idx + 1}/{len(features_to_test)}: {feature} ---\")\n            \n            # Generate configs for this feature\n            feature_configs = config_generator.generate_configs_for_features([feature])\n            \n            # Prioritize configs\n            prioritized_configs = config_selector.prioritize_configs(\n                feature_configs, \n                min(20, len(feature_configs))\n            )\n            \n            feature_best_score = -np.inf\n            \n            for config_idx, config in enumerate(prioritized_configs):\n                # Check runtime\n                elapsed_hours = (time.time() - self.start_time) / 3600\n                if elapsed_hours >= Config.MAX_RUNTIME_HOURS:\n                    print(f\"\\nReached maximum runtime of {Config.MAX_RUNTIME_HOURS} hours\")\n                    self._save_summary(n_completed, n_errors, n_skipped, best_score, best_config)\n                    return\n                \n                if config_idx % 5 == 0:\n                    print(f\"  Config {config_idx + 1}/{len(prioritized_configs)}\")\n                \n                # Run experiment\n                result = self.experiment_runner.run_experiment(train_df, test_df, config, config_selector)\n                \n                if result['success']:\n                    n_completed += 1\n                    val_score = result['mean_val_score']\n                    \n                    # Skip if severe overfitting\n                    if result.get('overfitting_flag', False) and result.get('train_val_gap', 0) > Config.MAX_TRAIN_VAL_GAP:\n                        print(f\"    Skipping due to overfitting (gap: {result['train_val_gap']:.4f})\")\n                        continue\n                    \n                    if val_score > feature_best_score:\n                        feature_best_score = val_score\n                    \n                    if val_score > best_score:\n                        best_score = val_score\n                        best_config = config\n                        print(f\"  🎯 New best score: {best_score:.4f} (improvement: {best_score - self.baseline_score:.4f})\")\n                        \n                elif result.get('reason') == 'already_tested':\n                    n_skipped += 1\n                else:\n                    n_errors += 1\n                \n                # Progress update\n                if (n_completed + n_errors + n_skipped) % 10 == 0:\n                    self._log_progress(n_completed, n_errors, n_skipped, best_score)\n            \n            feature_scores[feature] = feature_best_score\n            print(f\"  Best score for {feature}: {feature_best_score:.4f}\")\n        \n        # Summary\n        self._save_summary(n_completed, n_errors, n_skipped, best_score, best_config)\n    \n    def _log_progress(self, n_completed: int, n_errors: int, n_skipped: int, best_score: float):\n        elapsed_hours = (time.time() - self.start_time) / 3600\n        improvement = best_score - self.baseline_score if self.baseline_score else 0\n        \n        print(f\"\\n--- Progress Update ---\")\n        print(f\"Runtime: {elapsed_hours:.2f} hours\")\n        print(f\"Completed: {n_completed}, Skipped: {n_skipped}, Errors: {n_errors}\")\n        print(f\"Best score: {best_score:.4f} (improvement: {improvement:.4f})\")\n        print(\"-\" * 23)\n    \n    def _save_summary(self, n_completed: int, n_errors: int, n_skipped: int, \n                     best_score: float, best_config: Optional[ExperimentConfig]):\n        print(\"\\n\" + \"=\"*80)\n        print(\"PIPELINE COMPLETE\")\n        print(\"=\"*80)\n        print(f\"Total runtime: {(time.time() - self.start_time) / 3600:.2f} hours\")\n        print(f\"Experiments completed: {n_completed}\")\n        print(f\"Experiments skipped: {n_skipped}\")\n        print(f\"Experiments errored: {n_errors}\")\n        print(f\"Baseline score: {self.baseline_score:.4f}\")\n        print(f\"Best validation score: {best_score:.4f}\")\n        print(f\"Improvement: {best_score - self.baseline_score:.4f}\")\n        \n        if best_config:\n            print(f\"\\nBest configuration:\")\n            print(f\"  {best_config.get_description()}\")\n            print(f\"  Additional features: {best_config.additional_features}\")\n\ndef signal_handler(sig, frame):\n    print(\"\\n\\nReceived interrupt signal. Shutting down gracefully...\")\n    sys.exit(0)\n\ndef main():\n    signal.signal(signal.SIGINT, signal_handler)\n    \n    # Set random seeds\n    np.random.seed(42)\n    torch.manual_seed(42)\n    random.seed(42)\n    \n    # Run pipeline\n    pipeline = EnhancedCryptoPipeline()\n    pipeline.run()\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}