{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":12405352,"sourceType":"datasetVersion","datasetId":7823194},{"sourceId":249869065,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python\n# coding: utf-8\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.linear_model import Ridge, SGDRegressor\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import VotingRegressor\nimport os\nimport gc\nimport warnings\nfrom collections import defaultdict\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\ndef optimize_memory(df, verbose=True):\n    \"\"\"\n    Optimize memory usage by downcasting numeric types where possible.\n    \"\"\"\n    if verbose:\n        start_mem = df.memory_usage().sum() / 1024**2\n        print(f'Memory usage before optimization: {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != 'object':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    if verbose:\n        end_mem = df.memory_usage().sum() / 1024**2\n        print(f'Memory usage after optimization: {end_mem:.2f} MB')\n        print(f'Decreased by {100 * (start_mem - end_mem) / start_mem:.1f}%')\n    \n    return df\n\ndef get_feature_columns(df):\n    \"\"\"Get all feature columns including the extended list and new variables.\"\"\"\n    # Extended feature list provided\n    extended_features = [\n        'X727', 'X427', 'X288', 'X721', 'X312', 'X421', 'X471', 'X573', 'X780', 'X255',\n        'X144', 'X299', 'X301', 'X563', 'X737', 'X702', 'ask_qty', 'X507', 'X306', 'X501',\n        'X303', 'amihud_illiquidity', 'X586', 'X43', 'X517', 'X248', 'X137', 'X757', 'X196',\n        'X777', 'X280', 'X266', 'X689', 'X294', 'X492', 'X555', 'X731', 'X262', 'X576',\n        'X13', 'X518', 'X502', 'X558', 'pin_proxy', 'X6', 'X602', 'X695', 'X703', 'X413',\n        'X660', 'X37', 'X15', 'X310', 'X512', 'X362', 'X631', 'X214', 'X562', 'X488',\n        'X510', 'X256', 'X35', 'X128', 'X86', 'X170', 'X30', 'X265', 'X323', 'X559',\n        'X348', 'X130', 'X529', 'X20', 'X4', 'X90', 'X192', 'X91', 'X582', 'X99',\n        'X24', 'X317', 'X707', 'X653', 'X519', 'X557', 'X371', 'X415', 'X84', 'X83',\n        'order_toxicity', 'X360', 'X111', 'X699', 'X187', 'X591', 'X637', 'X567', 'X577',\n        'X313', 'X60', 'X671', 'X698', 'X701', 'X725', 'X292', 'X638', 'X741', 'X379',\n        'X700', 'X614', 'X676', 'X516', 'X697', 'X611', 'X311', 'X615', 'X706', 'X466',\n        'X571', 'X451', 'X17', 'X584', 'X436', 'X305', 'liquidity_consumption', 'X34', 'X282',\n        'X681', 'X7', 'X208', 'X41', 'X536', 'X548', 'X296', 'X776', 'X87', 'X40',\n        'X570', 'X539', 'X474', 'X753', 'X425', 'X217', 'X199', 'X18', 'X609', 'X21',\n        'X277', 'X279', 'X326', 'X540', 'X688', 'X553', 'X452', 'X738', 'X183', 'X759',\n        'bid_ask_ratio', 'X495', 'volume_participation', 'X715', 'X385', 'X291', 'X409', 'X112',\n        'X693', 'X102', 'X318', 'X705', 'X556', 'X547'\n    ]\n    \n    # Core features that are usually important\n    core_features = [\n        'X363', 'X405', 'X321', 'X175', 'X179', 'X197', 'X22', 'X181', 'X28', 'X169',\n        'X198', 'X173', 'X338', 'X344', 'X587', 'X450', 'X97', 'X52', 'X444', 'X598',\n        'X297', 'X138', 'X572', 'X343', 'X438', 'X459', 'X758', 'X25', 'buy_qty', \n        'sell_qty', 'volume', 'bid_qty'\n    ]\n    \n    # NEW features to add\n    new_features = [\n        'X363', 'X321', 'X405', 'X730', 'X523', 'X756', 'X589', 'X462', 'X779', 'log_liquidity',\n        'X25', 'X532', 'X520', 'X329', 'X383', 'X751', 'X535', 'X639', 'X596', 'X761',\n        'X145', 'X709', 'X173', 'X245', 'X168', 'X171', 'X241', 'X31', 'X105', 'X63',\n        'X263', 'X426', 'X286', 'X357', 'X399', 'X315', 'X468', 'X131', 'X647', 'log_spread',\n        'X752', 'X254', 'X592', 'X733', 'X636', 'X394', 'X527', 'X180', 'X367', 'X38',\n        'X634', 'X718', 'X387', 'X429', 'X345', 'X344', 'X253', 'X469', 'X446', 'X125',\n        'X760', 'X186', 'X711', 'X150', 'X661', 'X215', 'X403', 'X141', 'X771', 'X453',\n        'X401', 'X629', 'X616', 'X281', 'X432', 'X283', 'X244', 'X440', 'X430', 'X382',\n        'X175', 'X95', 'X444', 'X189', 'X55', 'X605', 'X663', 'X194', 'X439', 'X670',\n        'X483', 'X163', 'X376', 'X71', 'X650', 'X203', 'X8', 'X624', 'X160', 'X100',\n        'X14', 'X511', 'X59', 'X302', 'X81', 'X325', 'X514', 'X649', 'X447', 'X538',\n        'X443', 'X39', 'X343', 'X12', 'X678', 'X775', 'X498', 'X249', 'X42', 'X384',\n        'kyle_lambda', 'X349', 'X356', 'X2', 'X250', 'X397', 'X685', 'X568', 'X136', 'X496',\n        'X53', 'X66', 'X374', 'X590', 'X668', 'X585', 'X677', 'X667', 'X530', 'X28',\n        'X64', 'X407', 'X494', 'X770', 'X710', 'X526', 'X644', 'X167', 'X190', 'X723',\n        'X33', 'X579', 'X206'\n    ]\n    \n    # ADDITIONAL features requested by user\n    additional_features = [\n        'X525', 'X267', 'X166', 'X719', 'X489', 'X758', 'X652', 'X433', 'X778', 'X428',\n        'X617', 'X259', 'X633', 'X565', 'X364', 'depth_ratio', 'X550', 'X687', 'X610', 'X599',\n        'X717', 'X587', 'X143', 'X506', 'X546', 'X505', 'X159', 'X574', 'X278', 'X458',\n        'X1', 'X749', 'X155', 'X651', 'X470', 'X580', 'X445', 'X373', 'X82', 'X607',\n        'X298', 'X221', 'X388', 'X120', 'X391', 'X23', 'X679', 'X377', 'X767', 'X755',\n        'X566', 'X424', 'X438', 'X198', 'X300', 'X268', 'X434', 'X290', 'X368', 'X464',\n        'X119', 'X197', 'X597', 'X157', 'X485', 'X127', 'X101', 'X533', 'X235', 'X712',\n        'X154', 'X239', 'X10', 'X420', 'X449', 'X740', 'X227', 'X36', 'X358', 'X551',\n        'X528', 'X285', 'X335', 'X152', 'X110', 'X68', 'X713', 'X402', 'X370', 'X735',\n        'X200', 'X331', 'X473', 'X162', 'X213', 'X322', 'X289', 'X477', 'X113', 'X560',\n        'X672', 'X621', 'X682', 'X5', 'X72', 'X44', 'X419', 'buy_pressure', 'X242', 'volume',\n        'X472', 'X332', 'X441', 'buy_sell_ratio', 'pressure_ratio', 'X508', 'X594', 'X191',\n        'X261', 'X603', 'net_pressure', 'order_flow_imbalance', 'sell_pressure', 'X240', 'X673',\n        'X608', 'X509', 'X165', 'X720', 'X314', 'X522', 'X531', 'X625', 'bid_depth_ratio',\n        'X435', 'X293', 'X486', 'price_efficiency', 'X716', 'X627', 'X626', 'X169', 'X613',\n        'X680', 'X544', 'X115', 'X307', 'X665', 'X465', 'X347', 'X728', 'X70', 'log_volume',\n        'X340', 'X459', 'X56', 'X395', 'X354', 'X51', 'X732', 'X247', 'X324', 'X316',\n        'X76', 'X341', 'X739', 'X601', 'X386', 'X683', 'X149', 'X193', 'X628', 'X309',\n        'X351', 'X393'\n    ]\n    \n    # NEW: 100 Additional X features\n    additional_100_features = [\n        # Features from X1-X100 range not yet included\n        'X3', 'X9', 'X11', 'X16', 'X19', 'X26', 'X27', 'X29', 'X32', 'X45',\n        'X46', 'X47', 'X48', 'X49', 'X50', 'X54', 'X57', 'X58', 'X61', 'X62',\n        'X65', 'X67', 'X69', 'X73', 'X74', 'X75', 'X77', 'X78', 'X79', 'X80',\n        'X85', 'X88', 'X89', 'X92', 'X93', 'X94', 'X96', 'X98',\n        \n        # Features from X100-X300 range\n        'X103', 'X104', 'X106', 'X107', 'X108', 'X109', 'X114', 'X116', 'X117', 'X118',\n        'X121', 'X122', 'X123', 'X124', 'X126', 'X129', 'X132', 'X133', 'X134', 'X135',\n        'X139', 'X140', 'X142', 'X146', 'X147', 'X148', 'X151', 'X153', 'X156', 'X158',\n        'X161', 'X164', 'X172', 'X174', 'X176', 'X177', 'X178', 'X182', 'X184', 'X185',\n        'X188', 'X195', 'X201', 'X202', 'X204', 'X205', 'X207', 'X209', 'X210', 'X211',\n        'X212', 'X216', 'X218', 'X219', 'X220', 'X222', 'X223', 'X224', 'X225', 'X226',\n        'X228', 'X229', 'X230', 'X231', 'X232', 'X233', 'X234', 'X236', 'X237', 'X238',\n        'X243', 'X246', 'X251', 'X252', 'X257', 'X258', 'X260', 'X264', 'X269', 'X270',\n        'X271', 'X272', 'X273', 'X275', 'X276', 'X284', 'X287', 'X295', 'X304', 'X308',\n        'X319', 'X320', 'X327', 'X328', 'X330', 'X333', 'X334', 'X336', 'X337', 'X339'\n    ]\n    \n    # Combine all features and remove duplicates\n    all_features = list(dict.fromkeys(\n        extended_features + core_features + new_features + \n        additional_features + additional_100_features\n    ))\n    \n    # Filter to only available columns\n    available_features = [col for col in all_features if col in df.columns]\n    \n    print(f\"Found {len(available_features)} features out of {len(all_features)} requested\")\n    \n    return available_features\n\nclass IncrementalLagEnsemble:\n    \"\"\"Incrementally trains models on different lag configurations using SGD.\"\"\"\n    \n    def __init__(self, feature_batch_size=50, lag_batch_size=5, n_epochs=15):\n        self.feature_batch_size = feature_batch_size\n        self.lag_batch_size = lag_batch_size\n        self.n_epochs = n_epochs  # Increased to 15 epochs\n        self.models = {}\n        self.scalers = {}\n        self.feature_names = None\n        self.model_weights = {}\n        self.performance_history = defaultdict(list)\n        \n        # Define lag strategies with odd/even splits for many configurations\n        self.lag_strategies = {\n            # Original micro lags\n            'micro': [1, 2, 3, 4, 5],\n            'micro_odd': [1, 3, 5, 7, 9],\n            'micro_even': [2, 4, 6, 8, 10],\n            \n            # Ultra short with odd/even\n            'ultra_short': [6, 8, 10, 12, 15],\n            'ultra_short_odd': [7, 9, 11, 13, 15, 17],\n            'ultra_short_even': [6, 8, 10, 12, 14, 16],\n            \n            # Short with odd/even\n            'short': [20, 25, 30, 40, 50],\n            'short_odd': [21, 25, 31, 41, 51],\n            'short_even': [20, 24, 30, 40, 50],\n            \n            # Short medium with odd/even\n            'short_medium': [60, 75, 90, 105, 120],\n            'short_medium_odd': [61, 75, 91, 105, 121],\n            'short_medium_even': [60, 74, 90, 104, 120],\n            \n            # Medium with odd/even\n            'medium': [150, 180, 210, 240, 300],\n            'medium_odd': [151, 181, 211, 241, 301],\n            'medium_even': [150, 180, 210, 240, 300],\n            \n            # Medium long with odd/even\n            'medium_long': [360, 420, 480, 540, 600],\n            'medium_long_odd': [361, 421, 481, 541, 601],\n            'medium_long_even': [360, 420, 480, 540, 600],\n            \n            # Long with odd/even\n            'long': [720, 840, 960, 1080, 1200],\n            'long_odd': [721, 841, 961, 1081, 1201],\n            'long_even': [720, 840, 960, 1080, 1200],\n            \n            # Very long with odd/even\n            'very_long': [1440, 1800, 2160, 2520, 2880],\n            'very_long_odd': [1441, 1801, 2161, 2521, 2881],\n            'very_long_even': [1440, 1800, 2160, 2520, 2880],\n            \n            # Ultra long (keeping original only due to very large values)\n            'ultra_long': [3600, 4320, 5040, 5760, 7200],\n            \n            # NEW LAG STRATEGIES ADDED\n            # Nano lags - for ultra high-frequency patterns\n            'nano': [1, 2, 3],\n            'nano_extended': [1, 2, 3, 4, 5, 6, 7],\n            \n            # Mixed frequency - combines different time scales\n            'mixed_short_long': [5, 30, 120, 720, 2880],\n            'mixed_all_scales': [2, 10, 50, 240, 1200, 5760],\n            \n            # Prime number lags - often capture unique patterns\n            'prime_short': [2, 3, 5, 7, 11, 13, 17, 19, 23],\n            'prime_medium': [29, 31, 37, 41, 43, 47, 53, 59, 61],\n            'prime_long': [71, 73, 79, 83, 89, 97, 101, 103, 107]\n        }\n        \n    def create_lag_features_batch(self, df, lag_list):\n        \"\"\"Create lag features for a batch of lags.\"\"\"\n        lag_features = []\n        \n        for lag in lag_list:\n            lagged = df.shift(-lag)\n            lagged.columns = [f'{col}_lag_{lag}' for col in df.columns]\n            lag_features.append(lagged)\n        \n        result = pd.concat([df] + lag_features, axis=1)\n        result = result.fillna(0)\n        \n        return result\n    \n    def train_sgd_model(self, X, y, model_name):\n        \"\"\"Train SGD model incrementally with more epochs.\"\"\"\n        print(f\"  Training SGD model: {model_name} ({self.n_epochs} epochs)\")\n        \n        # Initialize model and scaler if not exists\n        if model_name not in self.models:\n            self.models[model_name] = SGDRegressor(\n                loss='huber',\n                penalty='elasticnet',\n                alpha=0.0001,\n                l1_ratio=0.15,\n                learning_rate='invscaling',\n                eta0=0.01,\n                power_t=0.25,\n                random_state=42,\n                warm_start=True,\n                max_iter=1000,\n                tol=1e-3\n            )\n            self.scalers[model_name] = StandardScaler()\n            \n        model = self.models[model_name]\n        scaler = self.scalers[model_name]\n        \n        # Train in epochs with smaller chunks for better convergence\n        chunk_size = 20000  # Slightly smaller chunks for more updates with more epochs\n        for epoch in range(self.n_epochs):\n            # Shuffle indices for each epoch\n            indices = np.random.permutation(len(X))\n            \n            for start_idx in range(0, len(X), chunk_size):\n                end_idx = min(start_idx + chunk_size, len(X))\n                \n                # Get shuffled chunk\n                chunk_indices = indices[start_idx:end_idx]\n                X_chunk = X.iloc[chunk_indices]\n                y_chunk = y[chunk_indices]\n                \n                # Scale\n                if start_idx == 0 and epoch == 0:\n                    X_scaled = scaler.fit_transform(X_chunk)\n                else:\n                    X_scaled = scaler.transform(X_chunk)\n                \n                # Partial fit\n                model.partial_fit(X_scaled, y_chunk)\n            \n            # Print progress every 3 epochs\n            if epoch % 3 == 0:\n                print(f\"    Epoch {epoch+1}/{self.n_epochs} completed\")\n        \n        return model\n    \n    def train_feature_batch(self, feature_batch, X_full, y, strategy_name, lag_list):\n        \"\"\"Train on a batch of features with specific lags.\"\"\"\n        print(f\"\\n  Processing feature batch ({len(feature_batch)} features) with {strategy_name} lags\")\n        \n        # Select feature batch\n        X_batch = X_full[feature_batch].copy()\n        \n        # Create lag features\n        X_with_lags = self.create_lag_features_batch(X_batch, lag_list)\n        \n        # Train SGD model\n        model_name = f\"{strategy_name}_{feature_batch[0]}_{feature_batch[-1]}\"\n        self.train_sgd_model(X_with_lags, y, model_name)\n        \n        # Clean up\n        del X_batch, X_with_lags\n        gc.collect()\n        \n    def fit(self, X, y):\n        \"\"\"Fit ensemble using incremental training.\"\"\"\n        print(\"Training Incremental Lag Ensemble with Extended Lag Configurations...\")\n        print(f\"Total epochs per model: {self.n_epochs}\")\n        \n        self.feature_names = X.columns.tolist()\n        n_features = len(self.feature_names)\n        \n        # Split features into batches\n        feature_batches = []\n        for i in range(0, n_features, self.feature_batch_size):\n            batch = self.feature_names[i:i+self.feature_batch_size]\n            feature_batches.append(batch)\n        \n        print(f\"Split {n_features} features into {len(feature_batches)} batches\")\n        print(f\"Total lag strategies (including new ones): {len(self.lag_strategies)}\")\n        \n        # Calculate total lag values\n        total_lags = sum(len(lags) for lags in self.lag_strategies.values())\n        print(f\"Total unique lag values: {total_lags}\")\n        \n        # Train models for each combination of feature batch and lag strategy\n        total_models = len(feature_batches) * len(self.lag_strategies)\n        model_count = 0\n        \n        for strategy_name, lag_list in self.lag_strategies.items():\n            print(f\"\\nTraining {strategy_name} strategy (lags: {lag_list})\")\n            \n            for batch_idx, feature_batch in enumerate(feature_batches):\n                model_count += 1\n                print(f\"Progress: {model_count}/{total_models} models\")\n                \n                self.train_feature_batch(feature_batch, X, y, strategy_name, lag_list)\n                \n                # Clean up periodically\n                if batch_idx % 2 == 0:\n                    gc.collect()\n        \n        # Initialize equal weights\n        for model_name in self.models:\n            self.model_weights[model_name] = 1.0 / len(self.models)\n        \n        print(f\"\\nTotal models trained: {len(self.models)}\")\n        \n    def predict_batch(self, X, feature_batch, strategy_name, lag_list):\n        \"\"\"Make predictions for a specific feature batch and lag strategy.\"\"\"\n        model_name = f\"{strategy_name}_{feature_batch[0]}_{feature_batch[-1]}\"\n        \n        if model_name not in self.models:\n            return None\n            \n        # Select features\n        X_batch = X[feature_batch].copy()\n        \n        # Create lag features\n        X_with_lags = self.create_lag_features_batch(X_batch, lag_list)\n        \n        # Scale and predict\n        X_scaled = self.scalers[model_name].transform(X_with_lags)\n        predictions = self.models[model_name].predict(X_scaled)\n        \n        # Clean up\n        del X_batch, X_with_lags, X_scaled\n        gc.collect()\n        \n        return predictions\n    \n    def predict(self, X):\n        \"\"\"Make ensemble predictions.\"\"\"\n        all_predictions = []\n        weights = []\n        \n        # Recreate feature batches\n        n_features = len(self.feature_names)\n        feature_batches = []\n        for i in range(0, n_features, self.feature_batch_size):\n            batch = self.feature_names[i:i+self.feature_batch_size]\n            if all(col in X.columns for col in batch):\n                feature_batches.append(batch)\n        \n        # Get predictions from each model\n        for strategy_name, lag_list in self.lag_strategies.items():\n            for feature_batch in feature_batches:\n                pred = self.predict_batch(X, feature_batch, strategy_name, lag_list)\n                if pred is not None:\n                    all_predictions.append(pred)\n                    model_name = f\"{strategy_name}_{feature_batch[0]}_{feature_batch[-1]}\"\n                    weights.append(self.model_weights.get(model_name, 1.0))\n        \n        # Weighted average\n        if all_predictions:\n            weights = np.array(weights) / np.sum(weights)\n            return np.average(all_predictions, axis=0, weights=weights)\n        else:\n            return np.zeros(len(X))\n\n# Main execution starts here\nprint(\"=\"*70)\nprint(\"ENHANCED CRYPTO PREDICTION WITH MORE EPOCHS AND NEW LAG STRATEGIES\")\nprint(\"=\"*70)\n\n# Set pandas options\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = None\n\n# Load training data\nprint(\"\\nLoading training data...\")\ntrain_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\nprint(f\"Training data shape: {train_df.shape}\")\n\n# Optimize memory\ntrain_df = optimize_memory(train_df, verbose=True)\n\n# Extract labels\ny_train = train_df['label'].to_numpy().astype(np.float32)\n\n# Select features\nfeature_cols = get_feature_columns(train_df)\nX_train = train_df[feature_cols].copy()\n\n# Clean up\ndel train_df\ngc.collect()\n\n# Further optimize X_train\nX_train = optimize_memory(X_train, verbose=True)\n\n# Create and train the ensemble with more epochs\nensemble = IncrementalLagEnsemble(\n    feature_batch_size=35,  # Slightly smaller batches due to more features\n    lag_batch_size=5,\n    n_epochs=15  # Increased to 15 epochs for better convergence\n)\nensemble.fit(X_train, y_train)\n\n# Clean up training data\ndel X_train, y_train\ngc.collect()\n\n# Load test data\nprint(\"\\n\" + \"=\"*50)\nprint(\"LOADING TEST DATA\")\nprint(\"=\"*50)\n\ntest_df = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\nprint(f\"Test data shape: {test_df.shape}\")\n\n# Optimize memory\ntest_df = optimize_memory(test_df, verbose=True)\n\n# Timestamp reconstruction\ntimestamp_recon_path = '/kaggle/input/the-order-of-the-test-rows-2/closest_rows.csv'\nuse_timestamp_reconstruction = os.path.exists(timestamp_recon_path)\n\nif use_timestamp_reconstruction:\n    print(\"\\nApplying timestamp reconstruction...\")\n    \n    t = pd.Series(pd.read_csv(timestamp_recon_path)['0'].to_numpy())\n    print(f\"Timestamps loaded: {len(t)}\")\n    \n    # Process timestamps\n    t -= 10080\n    t[t < 0] = 538149\n    \n    t = t.sort_values()\n    t[t <= len(t)] = np.arange(t[t <= len(t)].shape[0])\n    t = t.sort_index()\n    \n    t = pd.Series(np.arange(538150), index=t.to_numpy()).sort_index()\n    \n    # Sort test data\n    test_df = test_df.iloc[t.to_numpy()]\n    print(\"Test data sorted by reconstructed timestamps\")\nelse:\n    print(\"No timestamp reconstruction file found\")\n    t = pd.Series(np.arange(len(test_df)))\n\n# Select same features as training\nX_test = test_df[feature_cols].copy()\ndel test_df\ngc.collect()\n\n# Optimize test features\nX_test = optimize_memory(X_test, verbose=True)\n\n# Make predictions in chunks\nprint(\"\\n\" + \"=\"*50)\nprint(\"MAKING PREDICTIONS\")\nprint(\"=\"*50)\n\nchunk_size = 12000  # Slightly smaller chunks due to more models\nn_samples = len(X_test)\ny_pred = np.zeros(n_samples, dtype=np.float32)\n\nn_chunks = (n_samples + chunk_size - 1) // chunk_size\nprint(f\"Processing {n_chunks} chunks of size {chunk_size}\")\n\nfor i in range(0, n_samples, chunk_size):\n    end_idx = min(i + chunk_size, n_samples)\n    chunk_num = i // chunk_size + 1\n    \n    if chunk_num % 10 == 0:\n        print(f\"\\nChunk {chunk_num}/{n_chunks} (rows {i}-{end_idx})\")\n    \n    # Get chunk\n    X_chunk = X_test.iloc[i:end_idx]\n    \n    # Predict\n    y_pred[i:end_idx] = ensemble.predict(X_chunk).astype(np.float32)\n    \n    # Clean up periodically\n    if chunk_num % 5 == 0:\n        gc.collect()\n\n# Clean up test data\ndel X_test\ngc.collect()\n\n# Display statistics\nprint(\"\\n\" + \"=\"*50)\nprint(\"PREDICTION STATISTICS\")\nprint(\"=\"*50)\n\npred_series = pd.Series(y_pred)\nprint(pred_series.describe())\n\n# Create visualizations\nfig, axes = plt.subplots(2, 2, figsize=(12, 8))\n\n# Cumulative sum\naxes[0, 0].plot(np.cumsum(y_pred))\naxes[0, 0].set_title('Cumulative Predictions')\naxes[0, 0].set_xlabel('Index')\naxes[0, 0].set_ylabel('Cumulative Sum')\naxes[0, 0].grid(True, alpha=0.3)\n\n# Distribution\naxes[0, 1].hist(y_pred, bins=50, alpha=0.7, edgecolor='black')\naxes[0, 1].set_title('Prediction Distribution')\naxes[0, 1].set_xlabel('Value')\naxes[0, 1].set_ylabel('Count')\n\n# First 2000 predictions\naxes[1, 0].plot(y_pred[:2000], alpha=0.7)\naxes[1, 0].set_title('First 2000 Predictions')\naxes[1, 0].set_xlabel('Index')\naxes[1, 0].set_ylabel('Prediction')\n\n# Rolling mean and std\nwindow = 1000\nrolling_mean = pred_series.rolling(window).mean()\nrolling_std = pred_series.rolling(window).std()\n\naxes[1, 1].plot(rolling_mean, label='Mean')\naxes[1, 1].fill_between(\n    range(len(rolling_mean)),\n    rolling_mean - rolling_std,\n    rolling_mean + rolling_std,\n    alpha=0.3,\n    label='±1 Std'\n)\naxes[1, 1].set_title(f'Rolling Statistics (window={window})')\naxes[1, 1].set_xlabel('Index')\naxes[1, 1].set_ylabel('Value')\naxes[1, 1].legend()\n\nplt.tight_layout()\nplt.show()\n\n# Additional analysis plot\nfig, ax = plt.subplots(1, 1, figsize=(10, 6))\n\n# Prediction volatility over time\nwindow_sizes = [100, 500, 1000, 5000]\nfor window in window_sizes:\n    rolling_vol = pred_series.rolling(window).std()\n    ax.plot(rolling_vol, label=f'Window {window}', alpha=0.7)\n\nax.set_title('Prediction Volatility Over Time')\nax.set_xlabel('Index')\nax.set_ylabel('Rolling Standard Deviation')\nax.legend()\nax.grid(True, alpha=0.3)\nplt.show()\n\n# Prepare submission\nprint(\"\\n\" + \"=\"*50)\nprint(\"PREPARING SUBMISSION\")\nprint(\"=\"*50)\n\nsubmission = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\n\nif use_timestamp_reconstruction:\n    submission = submission.iloc[t.to_numpy()]\n    submission['prediction'] = y_pred\n    submission = submission.sort_index()\nelse:\n    submission['prediction'] = y_pred\n\n# Save submission\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission saved to 'submission.csv'\")\n\n# Display submission info\nprint(\"\\nSubmission preview:\")\nprint(submission.head())\nprint(f\"\\nSubmission shape: {submission.shape}\")\nprint(f\"Prediction range: [{submission['prediction'].min():.6f}, {submission['prediction'].max():.6f}]\")\nprint(f\"Mean: {submission['prediction'].mean():.6f}\")\nprint(f\"Std: {submission['prediction'].std():.6f}\")\n\n# Percentile information\npercentiles = [1, 5, 10, 25, 50, 75, 90, 95, 99]\nprint(\"\\nPrediction percentiles:\")\nfor p in percentiles:\n    value = np.percentile(submission['prediction'], p)\n    print(f\"  {p}th percentile: {value:.6f}\")\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"PROCESS COMPLETED SUCCESSFULLY!\")\nprint(\"=\"*50)\n\n# Summary\nprint(\"\\nModel Summary:\")\nprint(f\"- Total features used: {len(feature_cols)}\")\nprint(f\"- Feature batches: {len(feature_cols) // ensemble.feature_batch_size + 1}\")\nprint(f\"- Lag strategies (including new ones): {len(ensemble.lag_strategies)}\")\nprint(f\"- Total models: {len(ensemble.models)}\")\nprint(f\"- Model type: SGDRegressor with warm start\")\nprint(f\"- Training: Incremental with partial_fit ({ensemble.n_epochs} epochs)\")\nprint(f\"- Prediction: Weighted ensemble average\")\n\n# Count total unique lag values\nall_lags = set()\nfor lags in ensemble.lag_strategies.values():\n    all_lags.update(lags)\nprint(f\"- Total unique lag values: {len(all_lags)}\")\n\n# Show new lag strategies\nprint(\"\\nNew lag strategies added:\")\nprint(\"- nano: Ultra high-frequency patterns (1-3)\")\nprint(\"- nano_extended: Extended high-frequency (1-7)\")\nprint(\"- mixed_short_long: Mixed time scales (5-2880)\")\nprint(\"- mixed_all_scales: All time scales (2-5760)\")\nprint(\"- prime_short/medium/long: Prime number lags for unique patterns\")\n\nprint(\"\\nDone!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import numpy as np\n# import pandas as pd\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # Models\n# from lightgbm import LGBMRegressor\n# from xgboost import XGBRegressor\n# from catboost import CatBoostRegressor\n# from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, HistGradientBoostingRegressor, VotingRegressor\n# from sklearn.linear_model import Ridge, Lasso, ElasticNet, HuberRegressor, BayesianRidge, ARDRegression, Lars, LassoLars\n# from sklearn.svm import SVR, NuSVR\n# from sklearn.neural_network import MLPRegressor\n# from sklearn.kernel_ridge import KernelRidge\n\n# # Utilities\n# from sklearn.model_selection import KFold, TimeSeriesSplit, GroupKFold\n# from sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer, PowerTransformer\n# from sklearn.feature_selection import mutual_info_regression, f_regression, SelectKBest, RFE\n# from sklearn.isotonic import IsotonicRegression\n# from sklearn.cluster import KMeans, DBSCAN\n# from sklearn.decomposition import PCA, FastICA, TruncatedSVD\n# from scipy.stats import pearsonr, spearmanr, rankdata, kurtosis, skew\n# from scipy.optimize import minimize, differential_evolution\n# from scipy.signal import find_peaks\n# import lightgbm as lgb\n# import xgboost as xgb\n# import os\n# import gc\n# from typing import Dict, List, Tuple, Optional\n# import copy\n# from joblib import Parallel, delayed\n# from numba import jit\n# import multiprocessing as mp\n\n# print(\"Starting Enhanced DRW Crypto Market Prediction Pipeline...\")\n# print(\"=\" * 100)\n\n# # Enhanced Configuration\n# class CFG:\n#     # Paths\n#     train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n#     test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n#     sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n#     # Cross-validation\n#     n_folds = 7  # Increased for better validation\n#     random_state = 42\n#     use_gpu = False\n#     n_jobs = mp.cpu_count()\n    \n#     # Anti-overfitting parameters\n#     use_discriminator = True\n#     overfit_threshold = 0.12  # Stricter threshold\n#     confidence_decay = 0.85\n#     min_confidence = 0.25\n#     use_adversarial_validation = True\n    \n#     # Feature engineering settings\n#     n_proprietary_features = 80  # Increased\n#     n_interaction_features = 150  # Increased\n#     n_statistical_features = 60  # Increased\n#     n_temporal_features = 50  # Increased\n#     n_cluster_features = 30  # Increased\n#     n_pca_components = 50  # New\n#     n_ica_components = 20  # New\n    \n#     # Feature selection\n#     use_feature_selection = True\n#     feature_selection_threshold = 0.005  # More selective\n#     max_features = 300  # Increased\n#     use_recursive_elimination = True\n    \n#     # Important features from analysis (extended)\n#     important_x_features = [\n#         \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n#         \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n#         \"X758\", \"X296\", \"X611\", \"X780\", \"X451\", \"X25\", \"X591\", \"X673\", \"X274\", \"X286\",\n#         \"X388\", \"X601\", \"X294\", \"X675\", \"X741\", \"X88\", \"X199\", \"X312\", \"X723\", \"X444\"\n#     ]\n    \n#     # Ensemble settings\n#     use_stacking = True\n#     use_blending = True\n#     use_second_level_stacking = True  # New\n#     ensemble_top_n = 30  # Increased\n#     use_bayesian_optimization = True  # New\n    \n#     # Data augmentation\n#     use_data_augmentation = True\n#     augmentation_noise_level = 0.01\n#     augmentation_samples = 5000\n\n# # ============= Enhanced Memory Optimization =============\n# def reduce_mem_usage(df, name=\"\", aggressive=True):\n#     \"\"\"Reduce memory usage of dataframe with aggressive optimization\"\"\"\n#     print(f\"Optimizing memory for {name}...\")\n#     start_mem = df.memory_usage().sum() / 1024**2\n    \n#     for col in df.columns:\n#         col_type = df[col].dtype\n#         if col_type != object:\n#             c_min = df[col].min()\n#             c_max = df[col].max()\n            \n#             # Check for boolean columns\n#             if aggressive and df[col].nunique() == 2:\n#                 df[col] = df[col].astype(bool)\n#             elif str(col_type)[:3] == 'int':\n#                 if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n#                     df[col] = df[col].astype(np.int8)\n#                 elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n#                     df[col] = df[col].astype(np.int16)\n#                 elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n#                     df[col] = df[col].astype(np.int32)\n#             else:\n#                 # More aggressive float optimization\n#                 if aggressive and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n#                     df[col] = df[col].astype(np.float16)\n#                 elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n#                     df[col] = df[col].astype(np.float32)\n    \n#     end_mem = df.memory_usage().sum() / 1024**2\n#     print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n#     return df\n\n# # ============= Advanced Feature Engineering =============\n# @jit(nopython=True)\n# def fast_rolling_stats(arr, window):\n#     \"\"\"Fast rolling statistics using numba\"\"\"\n#     n = len(arr)\n#     rolling_mean = np.zeros(n)\n#     rolling_std = np.zeros(n)\n    \n#     for i in range(n):\n#         start = max(0, i - window + 1)\n#         window_data = arr[start:i+1]\n#         rolling_mean[i] = np.mean(window_data)\n#         rolling_std[i] = np.std(window_data) if len(window_data) > 1 else 0\n    \n#     return rolling_mean, rolling_std\n\n# def add_advanced_microstructure_features(df):\n#     \"\"\"Create advanced market microstructure features\"\"\"\n#     print(\"Engineering advanced microstructure features...\")\n    \n#     eps = 1e-8\n    \n#     # Existing features (keep all original features)\n#     df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n#     df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n#     df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + eps)\n#     df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    \n#     # Pressure indicators\n#     df['buying_pressure'] = df['buy_qty'] / (df['volume'] + eps)\n#     df['selling_pressure'] = df['sell_qty'] / (df['volume'] + eps)\n#     df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n    \n#     # Liquidity features\n#     df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n#     df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + eps)\n#     df['liquidity_ratio'] = df['total_liquidity'] / (df['volume'] + eps)\n    \n#     # Volume transformations\n#     df['log_volume'] = np.log1p(df['volume'])\n#     df['sqrt_volume'] = np.sqrt(df['volume'])\n#     df['volume_squared'] = df['volume'] ** 2\n#     df['volume_cubed'] = df['volume'] ** 3\n    \n#     # Market microstructure\n#     df['kyle_lambda'] = df['order_flow_imbalance'] / (df['sqrt_volume'] + eps)\n#     df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n    \n#     # Advanced microstructure features\n#     df['effective_spread'] = 2 * np.abs(df['order_flow_imbalance']) * df['bid_ask_spread']\n#     df['realized_spread'] = df['bid_ask_spread'] * df['vpin']\n#     df['price_impact'] = df['kyle_lambda'] * df['volume']\n#     df['trade_intensity'] = df['volume'] / (df['total_liquidity'] + eps)\n    \n#     # Additional advanced features\n#     df['amihud_illiquidity'] = np.abs(df['order_flow_imbalance']) / (df['volume'] + eps)\n#     df['roll_measure'] = 2 * np.sqrt(np.abs(df['order_flow_imbalance'] * df['bid_ask_spread']))\n#     df['depth_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n#     df['relative_spread'] = df['bid_ask_spread'] / (df['total_liquidity'] + eps)\n#     df['pin_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n#     df['order_flow_toxicity'] = df['vpin'] * df['kyle_lambda']\n#     df['market_stress'] = df['effective_spread'] * df['vpin']\n#     df['liquidity_tightness'] = df['liquidity_ratio'] * df['depth_ratio']\n    \n#     # New advanced features\n#     df['hasbrouck_lambda'] = df['order_flow_imbalance'] / (np.sqrt(df['volume'] * df['total_liquidity']) + eps)\n#     df['information_share'] = df['vpin'] ** 2 / (df['liquidity_imbalance'] ** 2 + eps)\n#     df['adverse_selection'] = df['effective_spread'] * df['order_flow_toxicity']\n#     df['inventory_risk'] = df['liquidity_imbalance'] * df['volume'] / (df['total_liquidity'] + eps)\n#     df['price_discovery'] = df['kyle_lambda'] * df['information_share']\n    \n#     # Microstructure interaction features\n#     df['spread_volume_interaction'] = df['bid_ask_spread'] * df['log_volume']\n#     df['imbalance_stress'] = df['order_flow_imbalance'] * df['market_stress']\n#     df['toxicity_impact'] = df['order_flow_toxicity'] * df['price_impact']\n#     df['liquidity_premium'] = df['amihud_illiquidity'] * df['effective_spread']\n    \n#     # Non-linear transformations\n#     df['ofi_sigmoid'] = 1 / (1 + np.exp(-df['order_flow_imbalance']))\n#     df['vpin_log'] = np.log1p(df['vpin'])\n#     df['kyle_sqrt'] = np.sqrt(np.abs(df['kyle_lambda']))\n#     df['stress_tanh'] = np.tanh(df['market_stress'])\n    \n#     # Handle infinities and NaNs\n#     numeric_cols = df.select_dtypes(include=[np.number]).columns\n#     for col in numeric_cols:\n#         df[col] = df[col].replace([np.inf, -np.inf], np.nan)\n#         df[col] = df[col].fillna(0)\n#         df[col] = np.clip(df[col], -1e8, 1e8)\n    \n#     return df\n\n# def add_enhanced_temporal_features(df, lookback_periods=[5, 10, 20, 60, 120]):\n#     \"\"\"Add enhanced rolling statistics and temporal patterns\"\"\"\n#     print(\"Creating enhanced temporal features...\")\n    \n#     # Key features for temporal analysis\n#     key_features = ['volume', 'order_flow_imbalance', 'kyle_lambda', 'vpin', \n#                    'bid_ask_spread', 'liquidity_imbalance', 'market_stress',\n#                    'order_flow_toxicity', 'price_impact', 'information_share']\n    \n#     for feature in key_features:\n#         if feature in df.columns:\n#             for period in lookback_periods:\n#                 # Use fast rolling stats for efficiency\n#                 if period <= 60:  # Only for smaller windows\n#                     values = df[feature].values\n#                     roll_mean, roll_std = fast_rolling_stats(values, period)\n                    \n#                     df[f'{feature}_roll_mean_{period}'] = roll_mean\n#                     df[f'{feature}_roll_std_{period}'] = roll_std\n#                     df[f'{feature}_roll_ratio_{period}'] = df[feature] / (roll_mean + 1e-8)\n                    \n#                     # Additional statistics for very important features\n#                     if feature in ['order_flow_imbalance', 'kyle_lambda', 'vpin'] and period <= 20:\n#                         df[f'{feature}_roll_max_{period}'] = df[feature].rolling(period, min_periods=1).max()\n#                         df[f'{feature}_roll_min_{period}'] = df[feature].rolling(period, min_periods=1).min()\n#                         df[f'{feature}_roll_range_{period}'] = df[f'{feature}_roll_max_{period}'] - df[f'{feature}_roll_min_{period}']\n#                         df[f'{feature}_roll_skew_{period}'] = df[feature].rolling(period, min_periods=1).skew()\n    \n#     # Exponentially weighted features with multiple spans\n#     for feature in key_features[:5]:  # Top 5 features\n#         if feature in df.columns:\n#             df[f'{feature}_ewm_12'] = df[feature].ewm(span=12, adjust=False).mean()\n#             df[f'{feature}_ewm_26'] = df[feature].ewm(span=26, adjust=False).mean()\n#             df[f'{feature}_ewm_60'] = df[feature].ewm(span=60, adjust=False).mean()\n            \n#             # MACD-like features\n#             df[f'{feature}_macd'] = df[f'{feature}_ewm_12'] - df[f'{feature}_ewm_26']\n#             df[f'{feature}_macd_signal'] = df[f'{feature}_macd'].ewm(span=9, adjust=False).mean()\n    \n#     # Enhanced lag features for important X variables\n#     important_x = CFG.important_x_features[:15]  # Top 15\n#     for x_feat in important_x:\n#         if x_feat in df.columns:\n#             for lag in [1, 2, 5, 10, 20]:\n#                 df[f'{x_feat}_lag_{lag}'] = df[x_feat].shift(lag)\n#                 df[f'{x_feat}_diff_{lag}'] = df[x_feat] - df[x_feat].shift(lag)\n                \n#                 # Percentage change\n#                 if lag <= 5:\n#                     df[f'{x_feat}_pct_change_{lag}'] = df[x_feat].pct_change(lag)\n    \n#     # Time-based features\n#     if 'timestamp' in df.columns:\n#         # Assuming timestamp is available, extract time features\n#         df['hour'] = pd.to_datetime(df['timestamp']).dt.hour\n#         df['minute'] = pd.to_datetime(df['timestamp']).dt.minute\n#         df['hour_sin'] = np.sin(2 * np.pi * df['hour'] / 24)\n#         df['hour_cos'] = np.cos(2 * np.pi * df['hour'] / 24)\n#         df['minute_sin'] = np.sin(2 * np.pi * df['minute'] / 60)\n#         df['minute_cos'] = np.cos(2 * np.pi * df['minute'] / 60)\n    \n#     # Fill NaN values\n#     df = df.fillna(method='bfill').fillna(0)\n    \n#     return df\n\n# def create_advanced_proprietary_features(df, n_features=80):\n#     \"\"\"Create advanced proprietary X variable combinations\"\"\"\n#     print(f\"Creating {n_features} advanced proprietary features...\")\n    \n#     # Get existing X features\n#     x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n    \n#     # Prioritize important features\n#     base_features = [f for f in CFG.important_x_features if f in df.columns][:30]\n    \n#     # Add high-variance features if needed\n#     if len(base_features) < 30:\n#         x_variances = df[x_features].var()\n#         high_var_features = x_variances.nlargest(30).index.tolist()\n#         for feat in high_var_features:\n#             if feat not in base_features:\n#                 base_features.append(feat)\n#                 if len(base_features) >= 30:\n#                     break\n    \n#     prop_idx = 1\n    \n#     # Advanced statistical combinations\n#     if len(base_features) >= 15:\n#         # Weighted statistics using feature importance\n#         weights = np.linspace(1, 0.5, len(base_features[:15]))\n#         weights = weights / weights.sum()\n        \n#         df[f'X_prop_{prop_idx}'] = np.average(df[base_features[:15]], weights=weights, axis=1)\n#         prop_idx += 1\n        \n#         # Harmonic mean\n#         df[f'X_prop_{prop_idx}'] = len(base_features[:10]) / (1 / (df[base_features[:10]] + 1e-8)).sum(axis=1)\n#         prop_idx += 1\n        \n#         # Geometric mean\n#         df[f'X_prop_{prop_idx}'] = np.exp(np.log(np.abs(df[base_features[:10]]) + 1).mean(axis=1))\n#         prop_idx += 1\n        \n#         # Trimmed mean (remove outliers)\n#         df[f'X_prop_{prop_idx}'] = df[base_features[:15]].apply(lambda x: x.sort_values()[2:-2].mean(), axis=1)\n#         prop_idx += 1\n        \n#         # Entropy-like measure\n#         df[f'X_prop_{prop_idx}'] = -(df[base_features[:10]] * np.log(np.abs(df[base_features[:10]]) + 1e-8)).sum(axis=1)\n#         prop_idx += 1\n        \n#         # Gini coefficient\n#         def gini(x):\n#             sorted_x = np.sort(x)\n#             n = len(x)\n#             index = np.arange(1, n + 1)\n#             return (2 * index - n - 1).dot(sorted_x) / (n * sorted_x.sum() + 1e-8)\n        \n#         df[f'X_prop_{prop_idx}'] = df[base_features[:10]].apply(gini, axis=1)\n#         prop_idx += 1\n    \n#     # Principal component features\n#     if len(base_features) >= 20:\n#         pca = PCA(n_components=5, random_state=42)\n#         pca_features = pca.fit_transform(df[base_features[:20]])\n#         for i in range(5):\n#             df[f'X_prop_{prop_idx}'] = pca_features[:, i]\n#             prop_idx += 1\n    \n#     # Advanced non-linear transformations\n#     for i in range(min(20, n_features - prop_idx + 1)):\n#         feat = base_features[i % len(base_features)]\n        \n#         if i < 5:\n#             # Sigmoid with different scales\n#             df[f'X_prop_{prop_idx}'] = 1 / (1 + np.exp(-df[feat] / (df[feat].std() + 1e-8)))\n#         elif i < 10:\n#             # Polynomial features\n#             df[f'X_prop_{prop_idx}'] = df[feat] ** 3 - 3 * df[feat]\n#         elif i < 15:\n#             # Trigonometric features\n#             df[f'X_prop_{prop_idx}'] = np.sin(df[feat] / (df[feat].std() + 1e-8))\n#         else:\n#             # Custom transformations\n#             df[f'X_prop_{prop_idx}'] = np.where(df[feat] > 0, np.log1p(df[feat]), -np.log1p(-df[feat]))\n#         prop_idx += 1\n    \n#     # Market regime interactions\n#     market_features = ['volume', 'order_flow_imbalance', 'kyle_lambda', 'vpin', \n#                       'market_stress', 'order_flow_toxicity', 'information_share']\n    \n#     for mkt_feat in market_features:\n#         if mkt_feat in df.columns and prop_idx <= n_features - 10:\n#             # Create market regimes\n#             mkt_quantiles = df[mkt_feat].quantile([0.25, 0.75])\n#             low_regime = df[mkt_feat] < mkt_quantiles[0.25]\n#             high_regime = df[mkt_feat] > mkt_quantiles[0.75]\n            \n#             for i in range(3):\n#                 x_feat = base_features[i % len(base_features)]\n                \n#                 # Regime-specific features\n#                 df[f'X_prop_{prop_idx}'] = df[x_feat] * low_regime\n#                 prop_idx += 1\n                \n#                 if prop_idx <= n_features:\n#                     df[f'X_prop_{prop_idx}'] = df[x_feat] * high_regime\n#                     prop_idx += 1\n                \n#                 if prop_idx > n_features:\n#                     break\n    \n#     # Complex feature interactions\n#     while prop_idx <= n_features and len(base_features) >= 3:\n#         i = (prop_idx - 1) % len(base_features)\n#         j = (i + 1) % len(base_features)\n#         k = (i + 2) % len(base_features)\n        \n#         feat1 = base_features[i]\n#         feat2 = base_features[j]\n#         feat3 = base_features[k]\n        \n#         # Three-way interaction with market condition\n#         market_condition = df['market_stress'] if 'market_stress' in df.columns else 1\n#         df[f'X_prop_{prop_idx}'] = df[feat1] * df[feat2] * np.tanh(df[feat3]) * market_condition\n#         prop_idx += 1\n    \n#     # Handle infinities and NaN\n#     prop_cols = [col for col in df.columns if col.startswith('X_prop_')]\n#     for col in prop_cols:\n#         df[col] = df[col].replace([np.inf, -np.inf], np.nan)\n#         df[col] = df[col].fillna(0)\n#         df[col] = np.clip(df[col], -1e6, 1e6)\n    \n#     print(f\"Created {prop_idx-1} proprietary features\")\n#     return df\n\n# def create_enhanced_interaction_features(df, selected_features, n_interactions=150):\n#     \"\"\"Create enhanced interaction features with market regime awareness\"\"\"\n#     print(f\"Creating {n_interactions} enhanced interaction features...\")\n    \n#     interaction_features = []\n#     feature_names = []\n    \n#     # Priority features\n#     important_x = [f for f in CFG.important_x_features if f in selected_features and f in df.columns][:20]\n#     market_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'liquidity_imbalance',\n#                       'bid_ask_spread', 'buying_pressure', 'volume', 'log_volume',\n#                       'market_stress', 'order_flow_toxicity', 'information_share', 'price_impact']\n#     priority_market = [f for f in market_features if f in selected_features and f in df.columns][:10]\n    \n#     interaction_count = 0\n    \n#     # Market regime definitions\n#     if 'market_stress' in df.columns:\n#         stress_median = df['market_stress'].median()\n#         high_stress = df['market_stress'] > stress_median\n#         low_stress = ~high_stress\n#     else:\n#         high_stress = low_stress = np.ones(len(df), dtype=bool)\n    \n#     # X features with market microstructure (regime-aware)\n#     for i, x_feat in enumerate(important_x[:15]):\n#         if interaction_count >= n_interactions // 3:\n#             break\n#         for j, market_feat in enumerate(priority_market[:6]):\n#             if interaction_count >= n_interactions // 3:\n#                 break\n            \n#             # Standard interaction\n#             interaction_features.append(df[x_feat] * df[market_feat])\n#             feature_names.append(f'{x_feat}_x_{market_feat}')\n#             interaction_count += 1\n            \n#             # Regime-specific interactions\n#             if interaction_count < n_interactions // 3:\n#                 interaction_features.append(df[x_feat] * df[market_feat] * high_stress)\n#                 feature_names.append(f'{x_feat}_x_{market_feat}_highstress')\n#                 interaction_count += 1\n    \n#     # Advanced polynomial features\n#     for i in range(min(15, len(important_x))):\n#         if interaction_count >= 2 * n_interactions // 3:\n#             break\n#         feat = important_x[i]\n        \n#         # Cubic\n#         interaction_features.append(df[feat] ** 3)\n#         feature_names.append(f'{feat}_cubed')\n#         interaction_count += 1\n        \n#         # Inverse\n#         if interaction_count < 2 * n_interactions // 3:\n#             interaction_features.append(1 / (np.abs(df[feat]) + 1e-8))\n#             feature_names.append(f'{feat}_inverse')\n#             interaction_count += 1\n        \n#         # Log absolute\n#         if interaction_count < 2 * n_interactions // 3:\n#             interaction_features.append(np.log1p(np.abs(df[feat])))\n#             feature_names.append(f'{feat}_log_abs')\n#             interaction_count += 1\n    \n#     # Complex market feature interactions\n#     market_triplets = [\n#         ('order_flow_imbalance', 'kyle_lambda', 'vpin'),\n#         ('liquidity_imbalance', 'market_stress', 'bid_ask_spread'),\n#         ('buying_pressure', 'selling_pressure', 'volume'),\n#         ('order_flow_toxicity', 'information_share', 'price_impact'),\n#         ('amihud_illiquidity', 'effective_spread', 'realized_spread')\n#     ]\n    \n#     for feat1, feat2, feat3 in market_triplets:\n#         if interaction_count >= n_interactions:\n#             break\n#         if all(f in df.columns for f in [feat1, feat2, feat3]):\n#             # Three-way interaction\n#             interaction_features.append(df[feat1] * df[feat2] * df[feat3])\n#             feature_names.append(f'{feat1}_{feat2}_{feat3}')\n#             interaction_count += 1\n            \n#             # Conditional interaction\n#             if interaction_count < n_interactions:\n#                 condition = df[feat3] > df[feat3].median()\n#                 interaction_features.append(df[feat1] * df[feat2] * condition)\n#                 feature_names.append(f'{feat1}_{feat2}_if_{feat3}_high')\n#                 interaction_count += 1\n    \n#     # Statistical interactions between X features\n#     if len(important_x) >= 5:\n#         for i in range(min(10, n_interactions - interaction_count)):\n#             # Random pairs for diversity\n#             idx1 = i % len(important_x)\n#             idx2 = (i + 3) % len(important_x)\n            \n#             if idx1 != idx2:\n#                 x1 = important_x[idx1]\n#                 x2 = important_x[idx2]\n                \n#                 # Harmonic mean\n#                 interaction_features.append(2 * df[x1] * df[x2] / (df[x1] + df[x2] + 1e-8))\n#                 feature_names.append(f'{x1}_harmonic_{x2}')\n#                 interaction_count += 1\n    \n#     # Create DataFrame\n#     interaction_df = pd.DataFrame(\n#         np.column_stack(interaction_features[:interaction_count]),\n#         columns=feature_names[:interaction_count],\n#         index=df.index\n#     )\n    \n#     # Handle infinities and NaN\n#     interaction_df = interaction_df.replace([np.inf, -np.inf], np.nan)\n#     interaction_df = interaction_df.fillna(0)\n#     interaction_df = np.clip(interaction_df, -1e6, 1e6)\n    \n#     print(f\"Created {interaction_count} interaction features\")\n#     return interaction_df\n\n# def create_advanced_cluster_features(X, n_clusters=30, features_to_cluster=None):\n#     \"\"\"Create advanced cluster-based meta-features with multiple algorithms\"\"\"\n#     print(f\"Creating {n_clusters} advanced cluster features...\")\n    \n#     if features_to_cluster is None:\n#         # Select diverse features for clustering\n#         x_features = [col for col in X.columns if col.startswith('X') and col[1:].isdigit()]\n#         market_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'market_stress']\n#         market_features = [f for f in market_features if f in X.columns]\n        \n#         features_to_cluster = x_features[:min(40, len(x_features))] + market_features\n    \n#     X_subset = X[features_to_cluster]\n    \n#     # Normalize for clustering\n#     scaler = StandardScaler()\n#     X_scaled = scaler.fit_transform(X_subset)\n    \n#     cluster_features = pd.DataFrame(index=X.index)\n    \n#     # KMeans clustering\n#     kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)\n#     kmeans_labels = kmeans.fit_predict(X_scaled)\n#     kmeans_distances = kmeans.transform(X_scaled)\n    \n#     # Add KMeans features\n#     for i in range(n_clusters):\n#         cluster_features[f'kmeans_dist_{i}'] = kmeans_distances[:, i]\n    \n#     cluster_features['kmeans_label'] = kmeans_labels\n#     cluster_features['kmeans_min_dist'] = kmeans_distances.min(axis=1)\n#     cluster_features['kmeans_dist_ratio'] = kmeans_distances.min(axis=1) / (kmeans_distances.max(axis=1) + 1e-8)\n    \n#     # Cluster statistics\n#     for i in range(min(10, n_clusters)):  # Top 10 clusters\n#         cluster_mask = kmeans_labels == i\n#         if cluster_mask.sum() > 0:\n#             cluster_features[f'kmeans_cluster_{i}_size_ratio'] = cluster_mask.sum() / len(X)\n    \n#     # DBSCAN for outlier detection\n#     try:\n#         dbscan = DBSCAN(eps=0.5, min_samples=5)\n#         dbscan_labels = dbscan.fit_predict(X_scaled[:min(10000, len(X_scaled))])  # Limit size for speed\n        \n#         if len(dbscan_labels) == len(X):\n#             cluster_features['dbscan_label'] = dbscan_labels\n#             cluster_features['is_outlier'] = (dbscan_labels == -1).astype(int)\n#         else:\n#             # Extend to full dataset if sampled\n#             cluster_features['is_outlier'] = 0\n#     except:\n#         cluster_features['is_outlier'] = 0\n    \n#     return cluster_features\n\n# def create_decomposition_features(X, n_components=50):\n#     \"\"\"Create features using various decomposition methods\"\"\"\n#     print(f\"Creating decomposition features with {n_components} components...\")\n    \n#     # Select numeric features\n#     numeric_features = X.select_dtypes(include=[np.number]).columns\n#     X_numeric = X[numeric_features]\n    \n#     # Normalize\n#     scaler = StandardScaler()\n#     X_scaled = scaler.fit_transform(X_numeric)\n    \n#     decomp_features = pd.DataFrame(index=X.index)\n    \n#     # PCA\n#     n_pca = min(n_components, X_scaled.shape[1] // 2)\n#     pca = PCA(n_components=n_pca, random_state=42)\n#     pca_features = pca.fit_transform(X_scaled)\n    \n#     for i in range(n_pca):\n#         decomp_features[f'pca_{i}'] = pca_features[:, i]\n    \n#     # Add explained variance ratios as features\n#     decomp_features['pca_explained_var_ratio'] = pca.explained_variance_ratio_[0]\n    \n#     # ICA (Independent Component Analysis)\n#     n_ica = min(20, X_scaled.shape[1] // 4)\n#     try:\n#         ica = FastICA(n_components=n_ica, random_state=42, max_iter=200)\n#         ica_features = ica.fit_transform(X_scaled)\n        \n#         for i in range(n_ica):\n#             decomp_features[f'ica_{i}'] = ica_features[:, i]\n#     except:\n#         print(\"ICA failed, skipping...\")\n    \n#     # TruncatedSVD for sparse-like features\n#     n_svd = min(30, X_scaled.shape[1] // 3)\n#     svd = TruncatedSVD(n_components=n_svd, random_state=42)\n#     svd_features = svd.fit_transform(X_scaled)\n    \n#     for i in range(min(10, n_svd)):  # Only keep top 10 SVD components\n#         decomp_features[f'svd_{i}'] = svd_features[:, i]\n    \n#     return decomp_features\n\n# # ============= Advanced Anti-Overfitting Framework =============\n# class AdvancedAntiOverfittingDiscriminator:\n#     \"\"\"Advanced discriminator with multiple overfitting detection methods\"\"\"\n    \n#     def __init__(self, overfit_threshold=0.12, confidence_decay=0.85, min_confidence=0.25):\n#         self.overfit_threshold = overfit_threshold\n#         self.confidence_decay = confidence_decay\n#         self.min_confidence = min_confidence\n#         self.model_metrics = {}\n#         self.adversarial_scores = {}\n        \n#     def evaluate_model_overfitting(self, model_name: str, train_scores: List[float], \n#                                   valid_scores: List[float], train_pred=None, \n#                                   valid_pred=None) -> Dict:\n#         \"\"\"Evaluate a model's overfitting characteristics with multiple metrics\"\"\"\n#         train_mean = np.mean(train_scores)\n#         valid_mean = np.mean(valid_scores)\n#         train_std = np.std(train_scores)\n#         valid_std = np.std(valid_scores)\n        \n#         # Calculate overfitting metrics\n#         overfit_gap = train_mean - valid_mean\n#         score_variance = np.var(valid_scores)\n#         score_stability = 1 / (1 + score_variance)\n#         consistency = 1 - abs(train_std - valid_std) / (train_std + valid_std + 1e-8)\n        \n#         # Advanced metrics\n#         if len(train_scores) > 1:\n#             score_trend = np.polyfit(range(len(valid_scores)), valid_scores, 1)[0]\n#             trend_stability = 1 / (1 + abs(score_trend))\n#         else:\n#             trend_stability = 1.0\n        \n#         # Prediction distribution analysis\n#         if train_pred is not None and valid_pred is not None:\n#             train_kurt = kurtosis(train_pred)\n#             valid_kurt = kurtosis(valid_pred)\n#             distribution_similarity = 1 / (1 + abs(train_kurt - valid_kurt))\n#         else:\n#             distribution_similarity = 1.0\n        \n#         # Determine if model is overfitting\n#         is_overfitting = (overfit_gap > self.overfit_threshold) or (consistency < 0.7)\n        \n#         # Calculate confidence score with multiple factors\n#         if is_overfitting:\n#             base_penalty = (overfit_gap / self.overfit_threshold) * (1 - self.confidence_decay)\n#             confidence = max(self.min_confidence, \n#                            1 - base_penalty * (2 - consistency) * (2 - trend_stability))\n#         else:\n#             confidence = min(1.0, \n#                            score_stability * consistency * trend_stability * \n#                            distribution_similarity * (1 + valid_mean))\n        \n#         metrics = {\n#             'train_score': train_mean,\n#             'valid_score': valid_mean,\n#             'train_std': train_std,\n#             'valid_std': valid_std,\n#             'overfit_gap': overfit_gap,\n#             'score_variance': score_variance,\n#             'score_stability': score_stability,\n#             'consistency': consistency,\n#             'trend_stability': trend_stability,\n#             'distribution_similarity': distribution_similarity,\n#             'is_overfitting': is_overfitting,\n#             'confidence': confidence\n#         }\n        \n#         self.model_metrics[model_name] = metrics\n#         return metrics\n    \n#     def adversarial_validation(self, X_train, X_test, sample_size=10000):\n#         \"\"\"Perform adversarial validation to detect distribution shift\"\"\"\n#         print(\"Performing adversarial validation...\")\n        \n#         # Sample for efficiency\n#         n_train = min(sample_size, len(X_train))\n#         n_test = min(sample_size, len(X_test))\n        \n#         X_train_sample = X_train.sample(n=n_train, random_state=42)\n#         X_test_sample = X_test.sample(n=n_test, random_state=42)\n        \n#         # Create labels\n#         y_adv = np.concatenate([np.zeros(n_train), np.ones(n_test)])\n#         X_adv = pd.concat([X_train_sample, X_test_sample])\n        \n#         # Train a simple model to distinguish train/test\n#         lgb_adv = LGBMRegressor(\n#             n_estimators=100,\n#             learning_rate=0.1,\n#             num_leaves=31,\n#             random_state=42,\n#             verbose=-1\n#         )\n        \n#         # Cross-validation\n#         kf = KFold(n_splits=3, shuffle=True, random_state=42)\n#         adv_scores = []\n        \n#         for train_idx, valid_idx in kf.split(X_adv):\n#             lgb_adv.fit(X_adv.iloc[train_idx], y_adv[train_idx])\n#             pred = lgb_adv.predict(X_adv.iloc[valid_idx])\n#             score = np.mean((pred > 0.5) == y_adv[valid_idx])\n#             adv_scores.append(score)\n        \n#         adv_score = np.mean(adv_scores)\n        \n#         # Score interpretation (0.5 = no distribution shift, 1.0 = complete shift)\n#         distribution_shift = 2 * abs(adv_score - 0.5)\n        \n#         self.adversarial_scores['distribution_shift'] = distribution_shift\n#         self.adversarial_scores['adv_accuracy'] = adv_score\n        \n#         print(f\"Adversarial validation accuracy: {adv_score:.3f}\")\n#         print(f\"Distribution shift score: {distribution_shift:.3f}\")\n        \n#         return distribution_shift\n    \n#     def get_adjusted_weights(self, model_names: List[str], base_weights: np.ndarray,\n#                            use_adversarial=True) -> np.ndarray:\n#         \"\"\"Adjust ensemble weights based on comprehensive overfitting analysis\"\"\"\n#         adjusted_weights = base_weights.copy()\n        \n#         # Apply model-specific adjustments\n#         for i, model_name in enumerate(model_names):\n#             if model_name in self.model_metrics:\n#                 confidence = self.model_metrics[model_name]['confidence']\n#                 adjusted_weights[i] *= confidence\n        \n#         # Apply global adversarial adjustment if available\n#         if use_adversarial and 'distribution_shift' in self.adversarial_scores:\n#             shift_penalty = 1 - self.adversarial_scores['distribution_shift'] * 0.5\n#             adjusted_weights *= shift_penalty\n        \n#         # Renormalize\n#         if adjusted_weights.sum() > 0:\n#             adjusted_weights /= adjusted_weights.sum()\n#         else:\n#             adjusted_weights = base_weights\n            \n#         return adjusted_weights\n\n# # ============= Enhanced Model Definitions =============\n# def get_enhanced_models():\n#     \"\"\"Get enhanced models with better regularization and diversity\"\"\"\n#     models = {\n#         # LightGBM variants (enhanced)\n#         'lgb_conservative': LGBMRegressor(\n#             n_estimators=800,\n#             learning_rate=0.008,\n#             num_leaves=31,\n#             max_depth=5,\n#             min_child_samples=60,\n#             subsample=0.5,\n#             colsample_bytree=0.5,\n#             reg_alpha=30,\n#             reg_lambda=30,\n#             min_split_gain=0.01,\n#             random_state=42,\n#             n_jobs=-1,\n#             verbose=-1\n#         ),\n        \n#         'lgb_balanced': LGBMRegressor(\n#             n_estimators=700,\n#             learning_rate=0.012,\n#             num_leaves=63,\n#             max_depth=7,\n#             min_child_samples=40,\n#             subsample=0.65,\n#             colsample_bytree=0.65,\n#             reg_alpha=15,\n#             reg_lambda=15,\n#             min_split_gain=0.005,\n#             random_state=42,\n#             n_jobs=-1,\n#             verbose=-1\n#         ),\n        \n#         'lgb_aggressive': LGBMRegressor(\n#             n_estimators=500,\n#             learning_rate=0.018,\n#             num_leaves=127,\n#             max_depth=9,\n#             min_child_samples=25,\n#             subsample=0.75,\n#             colsample_bytree=0.75,\n#             reg_alpha=8,\n#             reg_lambda=8,\n#             random_state=42,\n#             n_jobs=-1,\n#             verbose=-1\n#         ),\n        \n#         'lgb_dart': LGBMRegressor(\n#             n_estimators=600,\n#             learning_rate=0.015,\n#             num_leaves=95,\n#             max_depth=8,\n#             min_child_samples=35,\n#             subsample=0.7,\n#             colsample_bytree=0.7,\n#             reg_alpha=12,\n#             reg_lambda=12,\n#             boosting_type='dart',\n#             drop_rate=0.1,\n#             random_state=42,\n#             n_jobs=-1,\n#             verbose=-1\n#         ),\n        \n#         # XGBoost variants (enhanced)\n#         'xgb_conservative': XGBRegressor(\n#             n_estimators=1000,\n#             learning_rate=0.008,\n#             max_depth=4,\n#             subsample=0.5,\n#             colsample_bytree=0.5,\n#             gamma=3,\n#             reg_alpha=8,\n#             reg_lambda=8,\n#             min_child_weight=30,\n#             random_state=42,\n#             tree_method='gpu_hist' if CFG.use_gpu else 'hist',\n#             verbosity=0\n#         ),\n        \n#         'xgb_balanced': XGBRegressor(\n#             n_estimators=700,\n#             learning_rate=0.012,\n#             max_depth=6,\n#             subsample=0.65,\n#             colsample_bytree=0.65,\n#             gamma=1.5,\n#             reg_alpha=4,\n#             reg_lambda=4,\n#             min_child_weight=15,\n#             random_state=42,\n#             tree_method='gpu_hist' if CFG.use_gpu else 'hist',\n#             verbosity=0\n#         ),\n        \n#         'xgb_aggressive': XGBRegressor(\n#             n_estimators=600,\n#             learning_rate=0.018,\n#             max_depth=8,\n#             subsample=0.75,\n#             colsample_bytree=0.75,\n#             gamma=0.8,\n#             reg_alpha=2,\n#             reg_lambda=2,\n#             min_child_weight=8,\n#             random_state=42,\n#             tree_method='gpu_hist' if CFG.use_gpu else 'hist',\n#             verbosity=0\n#         ),\n        \n#         'xgb_dart': XGBRegressor(\n#             n_estimators=500,\n#             learning_rate=0.015,\n#             max_depth=7,\n#             subsample=0.7,\n#             colsample_bytree=0.7,\n#             gamma=1,\n#             reg_alpha=3,\n#             reg_lambda=3,\n#             min_child_weight=10,\n#             booster='dart',\n#             rate_drop=0.1,\n#             random_state=42,\n#             tree_method='gpu_hist' if CFG.use_gpu else 'hist',\n#             verbosity=0\n#         ),\n        \n#         # CatBoost variants (enhanced)\n#         'cat_conservative': CatBoostRegressor(\n#             iterations=1000,\n#             learning_rate=0.015,\n#             depth=4,\n#             l2_leaf_reg=25,\n#             border_count=128,\n#             random_state=42,\n#             verbose=False,\n#             task_type='GPU' if CFG.use_gpu else 'CPU'\n#         ),\n        \n#         'cat_balanced': CatBoostRegressor(\n#             iterations=700,\n#             learning_rate=0.025,\n#             depth=6,\n#             l2_leaf_reg=15,\n#             border_count=200,\n#             random_state=42,\n#             verbose=False,\n#             task_type='GPU' if CFG.use_gpu else 'CPU'\n#         ),\n        \n#         'cat_aggressive': CatBoostRegressor(\n#             iterations=500,\n#             learning_rate=0.04,\n#             depth=8,\n#             l2_leaf_reg=8,\n#             border_count=255,\n#             random_state=42,\n#             verbose=False,\n#             task_type='GPU' if CFG.use_gpu else 'CPU'\n#         ),\n        \n#         # Other tree-based models (enhanced)\n#         'rf_conservative': RandomForestRegressor(\n#             n_estimators=300,\n#             max_depth=10,\n#             min_samples_split=60,\n#             min_samples_leaf=25,\n#             max_features='sqrt',\n#             bootstrap=True,\n#             oob_score=True,\n#             random_state=42,\n#             n_jobs=-1\n#         ),\n        \n#         'extra_trees': ExtraTreesRegressor(\n#             n_estimators=300,\n#             max_depth=12,\n#             min_samples_split=50,\n#             min_samples_leaf=20,\n#             max_features='sqrt',\n#             bootstrap=False,\n#             random_state=42,\n#             n_jobs=-1\n#         ),\n        \n#         'hist_gb': HistGradientBoostingRegressor(\n#             max_iter=400,\n#             learning_rate=0.04,\n#             max_depth=8,\n#             min_samples_leaf=25,\n#             l2_regularization=1.5,\n#             max_bins=255,\n#             random_state=42\n#         ),\n        \n#         # Linear models (enhanced)\n#         'ridge': Ridge(alpha=10.0, random_state=42),\n#         'ridge_poly': Pipeline([\n#             ('poly', PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)),\n#             ('ridge', Ridge(alpha=100.0, random_state=42))\n#         ]),\n#         'lasso': Lasso(alpha=0.01, random_state=42, max_iter=3000),\n#         'elastic': ElasticNet(alpha=0.05, l1_ratio=0.7, random_state=42, max_iter=3000),\n#         'huber': HuberRegressor(epsilon=1.2, alpha=5.0, random_state=42, max_iter=200),\n#         'bayesian_ridge': BayesianRidge(\n#             alpha_1=1e-6, alpha_2=1e-6, \n#             lambda_1=1e-6, lambda_2=1e-6,\n#             n_iter=300\n#         ),\n#         'ard_regression': ARDRegression(\n#             alpha_1=1e-6, alpha_2=1e-6,\n#             lambda_1=1e-6, lambda_2=1e-6,\n#             n_iter=300\n#         ),\n#         'lars': Lars(n_nonzero_coefs=100),\n#         'lasso_lars': LassoLars(alpha=0.01, max_iter=500),\n        \n#         # Kernel methods\n#         'kernel_ridge': KernelRidge(alpha=1.0, kernel='rbf', gamma=0.01),\n        \n#         # SVM variants\n#         'svr_rbf': SVR(kernel='rbf', C=1.0, gamma='scale', epsilon=0.1),\n#         'svr_linear': SVR(kernel='linear', C=0.1, epsilon=0.01),\n#         'nu_svr': NuSVR(nu=0.5, C=1.0, kernel='rbf', gamma='scale'),\n        \n#         # Neural network\n#         'mlp': MLPRegressor(\n#             hidden_layer_sizes=(100, 50, 25),\n#             activation='relu',\n#             solver='adam',\n#             alpha=0.01,\n#             learning_rate='adaptive',\n#             learning_rate_init=0.001,\n#             max_iter=500,\n#             early_stopping=True,\n#             validation_fraction=0.1,\n#             random_state=42\n#         ),\n#     }\n    \n#     return models\n\n# # ============= Advanced Feature Selection =============\n# class AdvancedFeatureSelector:\n#     \"\"\"Advanced feature selection using multiple methods including RFE\"\"\"\n    \n#     def __init__(self):\n#         self.selected_features = None\n#         self.feature_scores = None\n#         self.rfe_support = None\n        \n#     def select_features(self, X, y, method='combined', n_features=None, use_rfe=True):\n#         \"\"\"Select features using multiple advanced methods\"\"\"\n#         print(f\"Selecting features using {method} method...\")\n        \n#         feature_scores = {}\n        \n#         # XGBoost importance with multiple metrics\n#         xgb_model = XGBRegressor(\n#             n_estimators=150, \n#             max_depth=6, \n#             learning_rate=0.1,\n#             random_state=42, \n#             n_jobs=-1,\n#             verbosity=0\n#         )\n#         xgb_model.fit(X, y)\n        \n#         # Get different importance types\n#         feature_scores['xgb_weight'] = pd.Series(\n#             xgb_model.get_booster().get_score(importance_type='weight'), \n#             index=X.columns\n#         ).fillna(0)\n#         feature_scores['xgb_gain'] = pd.Series(\n#             xgb_model.get_booster().get_score(importance_type='gain'), \n#             index=X.columns\n#         ).fillna(0)\n        \n#         # LightGBM importance\n#         lgb_model = LGBMRegressor(\n#             n_estimators=150,\n#             num_leaves=31,\n#             learning_rate=0.1,\n#             random_state=42,\n#             n_jobs=-1,\n#             verbose=-1\n#         )\n#         lgb_model.fit(X, y)\n        \n#         feature_scores['lgb_split'] = pd.Series(\n#             lgb_model.booster_.feature_importance(importance_type='split'),\n#             index=X.columns\n#         )\n#         feature_scores['lgb_gain'] = pd.Series(\n#             lgb_model.booster_.feature_importance(importance_type='gain'),\n#             index=X.columns\n#         )\n        \n#         # CatBoost importance\n#         cat_model = CatBoostRegressor(\n#             iterations=150,\n#             learning_rate=0.1,\n#             depth=6,\n#             random_state=42,\n#             verbose=False\n#         )\n#         cat_model.fit(X, y)\n#         feature_scores['catboost'] = pd.Series(\n#             cat_model.feature_importances_,\n#             index=X.columns\n#         )\n        \n#         # Mutual information\n#         mi_scores = mutual_info_regression(X, y, random_state=42, n_neighbors=5)\n#         feature_scores['mi'] = pd.Series(mi_scores, index=X.columns)\n        \n#         # F-statistic\n#         f_scores, _ = f_regression(X, y)\n#         feature_scores['f_stat'] = pd.Series(f_scores, index=X.columns)\n        \n#         # Correlation (both Pearson and Spearman)\n#         pearson_scores = pd.Series({\n#             col: abs(pearsonr(X[col], y)[0]) for col in X.columns\n#         })\n#         spearman_scores = pd.Series({\n#             col: abs(spearmanr(X[col], y)[0]) for col in X.columns\n#         })\n#         feature_scores['pearson'] = pearson_scores\n#         feature_scores['spearman'] = spearman_scores\n        \n#         # Normalize scores\n#         for key in feature_scores:\n#             scores = feature_scores[key]\n#             if scores.max() > scores.min():\n#                 feature_scores[key] = (scores - scores.min()) / (scores.max() - scores.min() + 1e-8)\n        \n#         # Combine scores with advanced weighting\n#         if method == 'combined':\n#             combined_scores = (\n#                 0.20 * feature_scores['xgb_gain'] +\n#                 0.15 * feature_scores['lgb_gain'] +\n#                 0.15 * feature_scores['catboost'] +\n#                 0.15 * feature_scores['mi'] +\n#                 0.10 * feature_scores['xgb_weight'] +\n#                 0.10 * feature_scores['f_stat'] +\n#                 0.08 * feature_scores['pearson'] +\n#                 0.07 * feature_scores['spearman']\n#             )\n#         else:\n#             combined_scores = feature_scores.get(method, feature_scores['xgb_gain'])\n        \n#         self.feature_scores = pd.DataFrame(feature_scores)\n#         self.feature_scores['combined'] = combined_scores\n#         self.feature_scores = self.feature_scores.sort_values('combined', ascending=False)\n        \n#         # Initial selection\n#         if n_features is None:\n#             n_features = min(CFG.max_features, max(200, int(len(X.columns) * 0.20)))\n        \n#         initial_features = self.feature_scores.head(n_features * 2).index.tolist()  # Get more for RFE\n        \n#         # Recursive Feature Elimination (optional but recommended)\n#         if use_rfe and CFG.use_recursive_elimination and len(initial_features) > n_features:\n#             print(f\"Applying RFE to refine selection from {len(initial_features)} to {n_features} features...\")\n            \n#             # Use a fast estimator for RFE\n#             rfe_estimator = LGBMRegressor(\n#                 n_estimators=50,\n#                 num_leaves=31,\n#                 learning_rate=0.1,\n#                 random_state=42,\n#                 n_jobs=-1,\n#                 verbose=-1\n#             )\n            \n#             rfe = RFE(\n#                 estimator=rfe_estimator,\n#                 n_features_to_select=n_features,\n#                 step=max(1, len(initial_features) // 20)\n#             )\n            \n#             X_subset = X[initial_features]\n#             rfe.fit(X_subset, y)\n            \n#             # Get RFE selected features\n#             rfe_features = [f for f, s in zip(initial_features, rfe.support_) if s]\n#             self.rfe_support = dict(zip(initial_features, rfe.support_))\n            \n#             # Combine RFE with importance scores\n#             self.selected_features = rfe_features\n#         else:\n#             self.selected_features = self.feature_scores.head(n_features).index.tolist()\n        \n#         # Always include critical features\n#         critical_features = [\n#             'order_flow_imbalance', 'kyle_lambda', 'vpin', 'volume', \n#             'bid_ask_spread', 'liquidity_imbalance', 'buying_pressure',\n#             'amihud_illiquidity', 'order_flow_toxicity', 'market_stress',\n#             'information_share', 'price_impact', 'adverse_selection'\n#         ]\n        \n#         # Also include top X features\n#         top_x_features = CFG.important_x_features[:10]\n        \n#         for feat in critical_features + top_x_features:\n#             if feat in X.columns and feat not in self.selected_features:\n#                 self.selected_features.append(feat)\n        \n#         print(f\"Selected {len(self.selected_features)} features\")\n#         return self.selected_features\n\n# # ============= Enhanced Training Functions =============\n# def create_advanced_time_weights(n_samples, method='exponential', decay_factor=0.95):\n#     \"\"\"Create advanced time-based sample weights\"\"\"\n#     positions = np.arange(n_samples)\n#     normalized_positions = positions / (n_samples - 1)\n    \n#     if method == 'exponential':\n#         weights = decay_factor ** (1 - normalized_positions)\n#     elif method == 'linear':\n#         weights = 0.5 + 0.5 * normalized_positions\n#     elif method == 'sigmoid':\n#         weights = 1 / (1 + np.exp(-10 * (normalized_positions - 0.7)))\n#     elif method == 'combined':\n#         # Combination of exponential and sigmoid\n#         exp_weights = decay_factor ** (1 - normalized_positions)\n#         sig_weights = 1 / (1 + np.exp(-10 * (normalized_positions - 0.7)))\n#         weights = 0.7 * exp_weights + 0.3 * sig_weights\n#     else:\n#         weights = np.ones(n_samples)\n    \n#     # Normalize weights\n#     weights = weights * n_samples / weights.sum()\n#     return weights\n\n# def train_model_with_advanced_tracking(model, X_train, y_train, X_valid, y_valid, \n#                                      sample_weight=None, model_name=\"\"):\n#     \"\"\"Train model with advanced tracking and early stopping\"\"\"\n    \n#     # Calculate feature importance weights for sample weighting\n#     if sample_weight is not None and model_name.startswith(('lgb', 'xgb', 'cat')):\n#         # Adjust sample weights based on prediction difficulty\n#         initial_pred = np.mean(y_train)  # Simple baseline\n#         residuals = np.abs(y_train - initial_pred)\n#         difficulty_weights = 1 + residuals / (residuals.max() + 1e-8)\n#         sample_weight = sample_weight * difficulty_weights\n#         sample_weight = sample_weight * len(sample_weight) / sample_weight.sum()\n    \n#     if model_name.startswith('lgb'):\n#         model.fit(\n#             X_train, y_train,\n#             sample_weight=sample_weight,\n#             eval_set=[(X_valid, y_valid)],\n#             callbacks=[lgb.early_stopping(80), lgb.log_evaluation(0)]\n#         )\n#     elif model_name.startswith('xgb'):\n#         model.fit(\n#             X_train, y_train,\n#             sample_weight=sample_weight,\n#             eval_set=[(X_valid, y_valid)],\n#             early_stopping_rounds=80,\n#             verbose=False\n#         )\n#     elif model_name.startswith('cat'):\n#         model.fit(\n#             X_train, y_train,\n#             sample_weight=sample_weight,\n#             eval_set=(X_valid, y_valid),\n#             early_stopping_rounds=80,\n#             verbose=False\n#         )\n#     else:\n#         # For sklearn models\n#         if hasattr(model, 'fit') and 'sample_weight' in model.fit.__code__.co_varnames:\n#             model.fit(X_train, y_train, sample_weight=sample_weight)\n#         else:\n#             model.fit(X_train, y_train)\n    \n#     # Get predictions\n#     train_pred = model.predict(X_train)\n#     valid_pred = model.predict(X_valid)\n    \n#     # Calculate multiple metrics\n#     train_pearson = pearsonr(y_train, train_pred)[0]\n#     valid_pearson = pearsonr(y_valid, valid_pred)[0]\n#     train_spearman = spearmanr(y_train, train_pred)[0]\n#     valid_spearman = spearmanr(y_valid, valid_pred)[0]\n    \n#     # Average of Pearson and Spearman for robustness\n#     train_score = (train_pearson + train_spearman) / 2\n#     valid_score = (valid_pearson + valid_spearman) / 2\n    \n#     return model, train_score, valid_score, valid_pred\n\n# # ============= Advanced Meta-Learning =============\n# class AdvancedMetaLearner:\n#     \"\"\"Advanced meta-learner with multiple optimization strategies\"\"\"\n    \n#     def __init__(self, base_predictions, y_true, test_predictions, discriminator=None):\n#         self.base_predictions = base_predictions\n#         self.y_true = y_true\n#         self.test_predictions = test_predictions\n#         self.n_models = len(base_predictions)\n#         self.discriminator = discriminator\n        \n#     def optimize_weights_bayesian(self, n_trials=100):\n#         \"\"\"Bayesian optimization for ensemble weights\"\"\"\n#         from scipy.stats import dirichlet\n        \n#         best_score = -np.inf\n#         best_weights = None\n        \n#         # Dirichlet distribution for weight sampling\n#         alpha = np.ones(self.n_models)\n        \n#         for _ in range(n_trials):\n#             # Sample weights from Dirichlet\n#             weights = dirichlet.rvs(alpha, size=1)[0]\n            \n#             # Evaluate\n#             weighted_pred = np.zeros_like(self.base_predictions","metadata":{"trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}