{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\n# Input data files are available in the read-only \"../input/\" directory\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Install required packages\nprint(\"Installing required packages...\")\n!pip install koolbox scikit-learn==1.5.2 prophet optuna --quiet\n\n# Import all required libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom prophet import Prophet\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import KFold, TimeSeriesSplit, cross_val_score\nfrom sklearn.linear_model import Ridge\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr, kurtosis, skew, entropy\nfrom xgboost import XGBRegressor\nfrom sklearn.base import clone\nfrom koolbox import Trainer\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport optuna\nimport joblib\nimport gc\nfrom scipy import stats\nfrom sklearn.preprocessing import RobustScaler, QuantileTransformer\nimport pickle\nfrom dataclasses import dataclass\nfrom typing import Dict, Tuple, List, Optional\nfrom collections import defaultdict\n\nwarnings.filterwarnings(\"ignore\")\n\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    target = \"label\"\n    n_folds = 10  # Increased for better stability analysis\n    n_stability_folds = 5  # Additional folds for stability testing\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 250\n\n@dataclass\nclass CompressionProfile:\n    \"\"\"Stores compression parameters for each feature\"\"\"\n    method: str\n    strength: float\n    median: float\n    mad: float\n    noise_level: float\n    outlier_score: float\n    optimal_compression: float\n    gradient_sensitivity: float\n    mutual_info_score: float\n\n@dataclass\nclass StabilityMetrics:\n    \"\"\"Stores stability metrics for a strategy\"\"\"\n    mean_cv_score: float\n    std_cv_score: float\n    stability_score: float  # mean - 2*std (higher is better)\n    fold_scores: List[float]\n    inter_fold_correlation: float\n\nclass IntelligentAdaptiveCompressor:\n    \"\"\"Advanced compressor with gradient-based optimization and mutual information analysis\"\"\"\n    \n    def __init__(self):\n        self.compression_profiles: Dict[str, CompressionProfile] = {}\n        self.label_profile: CompressionProfile = None\n        self.feature_importance: Dict[str, float] = {}\n        self.gradient_map: Dict[str, float] = {}\n        \n    def analyze_feature_advanced(self, data: np.ndarray, target: np.ndarray, \n                                feature_name: str) -> Dict[str, float]:\n        \"\"\"Advanced feature analysis including gradient sensitivity and mutual information\"\"\"\n        # Remove NaN values for analysis\n        mask = ~(np.isnan(data) | np.isnan(target))\n        clean_data = data[mask]\n        clean_target = target[mask]\n        \n        if len(clean_data) < 10:\n            return {\n                'noise_level': 0,\n                'outlier_score': 0,\n                'distribution_score': 0,\n                'gradient_sensitivity': 0,\n                'mutual_info_score': 0,\n                'optimal_compression': 0\n            }\n        \n        # Basic statistics\n        median = np.median(clean_data)\n        mad = np.median(np.abs(clean_data - median))\n        \n        # Noise level estimation\n        if len(clean_data) > 100:\n            # FFT-based noise estimation\n            fft = np.fft.fft(clean_data)\n            frequencies = np.abs(fft)\n            noise_level = np.sum(frequencies[len(frequencies)//2:]) / np.sum(frequencies)\n        else:\n            diffs = np.diff(np.sort(clean_data))\n            noise_level = np.std(diffs[diffs != 0]) / (mad + 1e-8)\n            \n        # Outlier score with robust estimation\n        if mad > 0:\n            z_scores = np.abs((clean_data - median) / mad)\n            outlier_score = np.mean(z_scores > 3)\n            extreme_outlier_score = np.mean(z_scores > 6)\n        else:\n            outlier_score = 0\n            extreme_outlier_score = 0\n            \n        # Distribution characteristics\n        try:\n            kurt = kurtosis(clean_data)\n            skewness = abs(skew(clean_data))\n            # Normalize distribution score\n            distribution_score = 1 / (1 + np.exp(-(kurt / 10 + skewness / 5)))\n        except:\n            distribution_score = 0.5\n            \n        # Gradient sensitivity - how much does target change with feature\n        try:\n            # Sort by feature value\n            sorted_idx = np.argsort(clean_data)\n            sorted_feature = clean_data[sorted_idx]\n            sorted_target = clean_target[sorted_idx]\n            \n            # Calculate local gradients\n            feature_diffs = np.diff(sorted_feature)\n            target_diffs = np.diff(sorted_target)\n            \n            # Avoid division by zero\n            mask = feature_diffs != 0\n            if np.any(mask):\n                gradients = np.abs(target_diffs[mask] / feature_diffs[mask])\n                gradient_sensitivity = np.percentile(gradients, 90)  # 90th percentile gradient\n            else:\n                gradient_sensitivity = 0\n        except:\n            gradient_sensitivity = 0\n            \n        # Mutual information score\n        try:\n            # Discretize for mutual information calculation\n            n_bins = min(20, len(clean_data) // 10)\n            feature_bins = pd.qcut(clean_data, n_bins, labels=False, duplicates='drop')\n            target_bins = pd.qcut(clean_target, n_bins, labels=False, duplicates='drop')\n            \n            # Calculate mutual information\n            from sklearn.metrics import mutual_info_score\n            mutual_info = mutual_info_score(feature_bins, target_bins)\n            \n            # Normalize by maximum possible entropy\n            max_entropy = np.log2(n_bins)\n            mutual_info_score_norm = mutual_info / max_entropy if max_entropy > 0 else 0\n        except:\n            mutual_info_score_norm = 0\n            \n        # Intelligent compression determination\n        # Features with high mutual information should be compressed less\n        # Features with high noise should be compressed more\n        # Features with extreme outliers need careful handling\n        \n        compression_factors = {\n            'noise': noise_level * 0.3,\n            'outliers': outlier_score * 0.2 + extreme_outlier_score * 0.1,\n            'distribution': distribution_score * 0.2,\n            'gradient': (1 - np.tanh(gradient_sensitivity / 10)) * 0.1,  # Less compression for sensitive features\n            'mutual_info': (1 - mutual_info_score_norm) * 0.3  # Less compression for informative features\n        }\n        \n        optimal_compression = np.clip(sum(compression_factors.values()), 0, 0.8)\n        \n        return {\n            'median': median,\n            'mad': mad,\n            'noise_level': noise_level,\n            'outlier_score': outlier_score,\n            'distribution_score': distribution_score,\n            'gradient_sensitivity': gradient_sensitivity,\n            'mutual_info_score': mutual_info_score_norm,\n            'optimal_compression': optimal_compression,\n            'compression_factors': compression_factors\n        }\n    \n    def fit(self, X: pd.DataFrame, y: pd.Series, \n            feature_compression_override: Dict[str, float] = None,\n            label_compression: float = 0.0,\n            use_gradient_optimization: bool = True):\n        \"\"\"Learn compression parameters with gradient optimization\"\"\"\n        \n        print(\"Learning intelligent adaptive compression parameters...\")\n        \n        # Calculate feature importance using random forest\n        if use_gradient_optimization:\n            from sklearn.ensemble import RandomForestRegressor\n            rf = RandomForestRegressor(n_estimators=50, random_state=42, n_jobs=-1)\n            rf.fit(X, y)\n            self.feature_importance = dict(zip(X.columns, rf.feature_importances_))\n        \n        # Analyze each feature\n        for col in X.columns:\n            analysis = self.analyze_feature_advanced(X[col].values, y.values, col)\n            \n            # Use override if provided, otherwise use adaptive compression\n            if feature_compression_override and col in feature_compression_override:\n                compression = feature_compression_override[col]\n            else:\n                # Adjust compression based on feature importance\n                base_compression = analysis['optimal_compression']\n                if use_gradient_optimization and col in self.feature_importance:\n                    importance_factor = 1 - self.feature_importance[col] / max(self.feature_importance.values())\n                    compression = base_compression * (0.5 + 0.5 * importance_factor)\n                else:\n                    compression = base_compression\n            \n            # Select method based on comprehensive analysis\n            if analysis['outlier_score'] > 0.15 or analysis['distribution_score'] > 0.7:\n                method = 'robust_tanh'  # Custom robust version\n            elif abs(analysis.get('skewness', 0)) > 2:\n                method = 'log'\n            elif analysis['gradient_sensitivity'] > 5:\n                method = 'soft_clip'  # Gentle clipping for sensitive features\n            else:\n                method = 'sigmoid'\n            \n            self.compression_profiles[col] = CompressionProfile(\n                method=method,\n                strength=compression,\n                median=analysis['median'],\n                mad=analysis['mad'],\n                noise_level=analysis['noise_level'],\n                outlier_score=analysis['outlier_score'],\n                optimal_compression=analysis['optimal_compression'],\n                gradient_sensitivity=analysis['gradient_sensitivity'],\n                mutual_info_score=analysis['mutual_info_score']\n            )\n            \n            print(f\"  {col}: {method} compression at {compression:.3f} \"\n                  f\"(noise={analysis['noise_level']:.3f}, MI={analysis['mutual_info_score']:.3f}, \"\n                  f\"gradient={analysis['gradient_sensitivity']:.3f})\")\n        \n        # Analyze label if provided\n        if y is not None and label_compression > 0:\n            # For labels, use very conservative compression\n            label_analysis = self.analyze_feature_advanced(y.values, y.values, 'label')\n            self.label_profile = CompressionProfile(\n                method='soft_clip',  # Always use soft clipping for labels\n                strength=label_compression * 0.5,  # Reduce strength for safety\n                median=label_analysis['median'],\n                mad=label_analysis['mad'],\n                noise_level=label_analysis['noise_level'],\n                outlier_score=label_analysis['outlier_score'],\n                optimal_compression=label_compression,\n                gradient_sensitivity=0,\n                mutual_info_score=1\n            )\n            print(f\"  Label: soft_clip compression at {label_compression * 0.5:.3f}\")\n    \n    def transform(self, X: pd.DataFrame, y: pd.Series = None) -> Tuple[pd.DataFrame, pd.Series]:\n        \"\"\"Apply learned compression with advanced methods\"\"\"\n        X_compressed = X.copy()\n        \n        # Compress each feature using its profile\n        for col in X.columns:\n            if col in self.compression_profiles:\n                profile = self.compression_profiles[col]\n                X_compressed[col] = self._compress_data_advanced(\n                    X[col].values,\n                    profile\n                )\n        \n        # Compress labels if profile exists\n        y_compressed = y\n        if y is not None and self.label_profile is not None:\n            y_compressed = pd.Series(\n                self._compress_data_advanced(y.values, self.label_profile),\n                index=y.index\n            )\n        \n        return X_compressed, y_compressed\n    \n    def _compress_data_advanced(self, data: np.ndarray, profile: CompressionProfile) -> np.ndarray:\n        \"\"\"Apply compression using advanced methods\"\"\"\n        # Handle NaN values\n        nan_mask = np.isnan(data)\n        data_clean = data[~nan_mask]\n        \n        if len(data_clean) == 0 or profile.strength == 0:\n            return data\n        \n        # Normalize using stored statistics\n        if profile.mad > 0:\n            normalized = (data - profile.median) / (profile.mad * 6)\n        else:\n            return data\n        \n        # Apply compression method\n        if profile.method == 'robust_tanh':\n            # Robust tanh with outlier handling\n            alpha = 1 - profile.strength\n            compressed = np.where(\n                np.abs(normalized) < 3,\n                np.tanh(normalized * alpha),\n                np.sign(normalized) * (0.995 + 0.005 * np.tanh((np.abs(normalized) - 3) * alpha))\n            )\n        elif profile.method == 'soft_clip':\n            # Soft clipping - gentle compression at extremes\n            threshold = 2 * (1 - profile.strength + 0.1)\n            compressed = np.where(\n                np.abs(normalized) < threshold,\n                normalized,\n                np.sign(normalized) * (threshold + np.log1p(np.abs(normalized) - threshold) * 0.5)\n            )\n        elif profile.method == 'tanh':\n            compressed = np.tanh(normalized * (1 - profile.strength))\n        elif profile.method == 'sigmoid':\n            compressed = 2 / (1 + np.exp(-normalized * (1 - profile.strength) * 2)) - 1\n        elif profile.method == 'arctan':\n            compressed = (2/np.pi) * np.arctan(normalized * (1 - profile.strength) * np.pi/2)\n        elif profile.method == 'log':\n            sign = np.sign(normalized)\n            abs_norm = np.abs(normalized)\n            compressed = sign * np.log1p(abs_norm * (1 - profile.strength)) / np.log1p(1 - profile.strength)\n        \n        # Scale back\n        result = compressed * (profile.mad * 6) + profile.median\n        \n        # Preserve NaN values\n        if nan_mask.any():\n            full_result = np.full_like(data, np.nan, dtype=np.float64)\n            full_result[~nan_mask] = result[~nan_mask]\n            return full_result\n        \n        return result\n    \n    def inverse_transform_predictions(self, predictions: np.ndarray) -> np.ndarray:\n        \"\"\"Inverse transform predictions if labels were compressed\"\"\"\n        if self.label_profile is None:\n            return predictions\n        \n        # Approximate inverse transformation\n        # This is a simplified version - in practice, you might want to fit an inverse function\n        profile = self.label_profile\n        \n        # Normalize predictions\n        normalized = (predictions - profile.median) / (profile.mad * 6)\n        \n        # Apply approximate inverse based on method\n        if profile.method == 'soft_clip':\n            # Inverse soft clip\n            threshold = 2 * (1 - profile.strength + 0.1)\n            inverse = np.where(\n                np.abs(normalized) < threshold,\n                normalized,\n                np.sign(normalized) * (threshold + np.expm1(np.abs(normalized) - threshold) * 2)\n            )\n        else:\n            # For other methods, use simple scaling\n            inverse = normalized / (1 - profile.strength * 0.5)\n        \n        # Scale back\n        return inverse * (profile.mad * 6) + profile.median\n    \n    def save(self, filepath: str):\n        \"\"\"Save compression profiles\"\"\"\n        with open(filepath, 'wb') as f:\n            pickle.dump({\n                'compression_profiles': self.compression_profiles,\n                'label_profile': self.label_profile,\n                'feature_importance': self.feature_importance\n            }, f)\n    \n    def load(self, filepath: str):\n        \"\"\"Load compression profiles\"\"\"\n        with open(filepath, 'rb') as f:\n            data = pickle.load(f)\n            self.compression_profiles = data['compression_profiles']\n            self.label_profile = data['label_profile']\n            self.feature_importance = data.get('feature_importance', {})\n\ndef calculate_stability_metrics(scores_dict: Dict[str, List[float]]) -> StabilityMetrics:\n    \"\"\"Calculate stability metrics across folds\"\"\"\n    all_scores = []\n    for model_scores in scores_dict.values():\n        all_scores.extend(model_scores)\n    \n    mean_score = np.mean(all_scores)\n    std_score = np.std(all_scores)\n    \n    # Calculate inter-fold correlation\n    fold_predictions = defaultdict(list)\n    for model, scores in scores_dict.items():\n        for i, score in enumerate(scores):\n            fold_predictions[i].append(score)\n    \n    # Average correlation between fold results\n    correlations = []\n    folds = list(fold_predictions.keys())\n    for i in range(len(folds)):\n        for j in range(i+1, len(folds)):\n            corr = np.corrcoef(fold_predictions[folds[i]], fold_predictions[folds[j]])[0, 1]\n            if not np.isnan(corr):\n                correlations.append(corr)\n    \n    inter_fold_corr = np.mean(correlations) if correlations else 0\n    \n    return StabilityMetrics(\n        mean_cv_score=mean_score,\n        std_cv_score=std_score,\n        stability_score=mean_score - 2 * std_score,  # Conservative stability metric\n        fold_scores=all_scores,\n        inter_fold_correlation=inter_fold_corr\n    )\n\ndef reduce_mem_usage(dataframe, dataset):    \n    print('Reducing memory usage for:', dataset)\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n\n    return dataframe\n\ndef _pearsonr(y_true, y_pred):\n    return pearsonr(y_true, y_pred)[0]\n\n# Define model parameters with stability-focused tuning\nlgbm_params = {\n    \"boosting_type\": \"gbdt\",\n    \"colsample_bytree\": 0.5625888953382505,\n    \"learning_rate\": 0.029312951475451557,\n    \"min_child_samples\": 63,\n    \"min_child_weight\": 0.11456572852335424,\n    \"n_estimators\": 126,\n    \"n_jobs\": -1,\n    \"num_leaves\": 37,\n    \"random_state\": 42,\n    \"reg_alpha\": 85.2476527854083,\n    \"reg_lambda\": 99.38305361388907,\n    \"subsample\": 0.450669817684892,\n    \"verbose\": -1,\n    \"feature_fraction_seed\": 42,  # For reproducibility\n    \"bagging_seed\": 42\n}\n\nlgbm_goss_params = {\n    \"boosting_type\": \"goss\",\n    \"colsample_bytree\": 0.34695458228489784,\n    \"learning_rate\": 0.031023014900595287,\n    \"min_child_samples\": 30,\n    \"min_child_weight\": 0.4727729225033618,\n    \"n_estimators\": 220,\n    \"n_jobs\": -1,\n    \"num_leaves\": 58,\n    \"random_state\": 42,\n    \"reg_alpha\": 38.665994901468224,\n    \"reg_lambda\": 92.76991677464294,\n    \"subsample\": 0.4810891284493255,\n    \"verbose\": -1,\n    \"feature_fraction_seed\": 42,\n    \"bagging_seed\": 42\n}\n\nxgb_params = {\n    \"colsample_bylevel\": 0.4778015829774066,\n    \"colsample_bynode\": 0.362764358742407,\n    \"colsample_bytree\": 0.7107423488010493,\n    \"gamma\": 1.7094857725240398,\n    \"learning_rate\": 0.02213323588455387,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 39.352415706891264,\n    \"reg_lambda\": 75.44843704068275,\n    \"subsample\": 0.06566669853471274,\n    \"verbosity\": 0,\n    \"seed\": 42\n}\n\n# GANDALF Model Implementation\nfrom sklearn.base import BaseEstimator, RegressorMixin\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nclass GANDALF(BaseEstimator, RegressorMixin):\n    def __init__(self, n_estimators=100, learning_rate=0.01, max_depth=5, \n                 feature_fraction=0.8, bagging_fraction=0.8, lambda_reg=1.0,\n                 min_data_in_leaf=20, num_iterations=100, random_state=42):\n        self.n_estimators = n_estimators\n        self.learning_rate = learning_rate\n        self.max_depth = max_depth\n        self.feature_fraction = feature_fraction\n        self.bagging_fraction = bagging_fraction\n        self.lambda_reg = lambda_reg\n        self.min_data_in_leaf = min_data_in_leaf\n        self.num_iterations = num_iterations\n        self.random_state = random_state\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        torch.manual_seed(random_state)  # For reproducibility\n        \n    def _build_network(self, input_dim):\n        \"\"\"Build the neural network architecture for GANDALF\"\"\"\n        class GANDALFNet(nn.Module):\n            def __init__(self, input_dim, hidden_dims=[256, 128, 64]):\n                super(GANDALFNet, self).__init__()\n                \n                layers = []\n                prev_dim = input_dim\n                \n                for hidden_dim in hidden_dims:\n                    layers.extend([\n                        nn.Linear(prev_dim, hidden_dim),\n                        nn.BatchNorm1d(hidden_dim),\n                        nn.ReLU(),\n                        nn.Dropout(0.3)\n                    ])\n                    prev_dim = hidden_dim\n                \n                layers.append(nn.Linear(prev_dim, 1))\n                \n                self.network = nn.Sequential(*layers)\n                \n            def forward(self, x):\n                return self.network(x)\n        \n        return GANDALFNet(input_dim).to(self.device)\n    \n    def fit(self, X, y):\n        # Convert to tensors\n        X_tensor = torch.FloatTensor(X.values if hasattr(X, 'values') else X).to(self.device)\n        y_tensor = torch.FloatTensor(y.values if hasattr(y, 'values') else y).reshape(-1, 1).to(self.device)\n        \n        # Build network\n        self.model = self._build_network(X_tensor.shape[1])\n        \n        # Create dataset and dataloader\n        dataset = TensorDataset(X_tensor, y_tensor)\n        dataloader = DataLoader(dataset, batch_size=1024, shuffle=True)\n        \n        # Optimizer\n        optimizer = optim.AdamW(self.model.parameters(), lr=self.learning_rate, weight_decay=self.lambda_reg)\n        criterion = nn.MSELoss()\n        \n        # Training loop\n        self.model.train()\n        for epoch in range(self.num_iterations):\n            total_loss = 0\n            for batch_X, batch_y in dataloader:\n                optimizer.zero_grad()\n                outputs = self.model(batch_X)\n                loss = criterion(outputs, batch_y)\n                loss.backward()\n                optimizer.step()\n                total_loss += loss.item()\n            \n            if (epoch + 1) % 20 == 0:\n                print(f'Epoch [{epoch+1}/{self.num_iterations}], Loss: {total_loss/len(dataloader):.4f}')\n        \n        return self\n    \n    def predict(self, X):\n        self.model.eval()\n        X_tensor = torch.FloatTensor(X.values if hasattr(X, 'values') else X).to(self.device)\n        \n        with torch.no_grad():\n            predictions = self.model(X_tensor).cpu().numpy().flatten()\n        \n        return predictions\n\n# AutoEncoder MLP\nfrom sklearn.base import BaseEstimator, RegressorMixin\nimport tensorflow as tf\nimport numpy as np\n\nclass AutoEncoderMLP(BaseEstimator, RegressorMixin):\n    def __init__(self, num_columns, hidden_units, dropout_rates, lr=1e-3, seed=42):\n        self.num_columns = num_columns\n        self.hidden_units = hidden_units\n        self.dropout_rates = dropout_rates\n        self.lr = lr\n        self.seed = seed\n        tf.random.set_seed(seed)\n        self.model = self._build_model()\n    \n    def _build_model(self):\n        inp = tf.keras.layers.Input(shape=(self.num_columns,))\n        x0 = tf.keras.layers.BatchNormalization()(inp)\n\n        encoder = tf.keras.layers.GaussianNoise(self.dropout_rates[0])(x0)\n        encoder = tf.keras.layers.Dense(self.hidden_units[0])(encoder)\n        encoder = tf.keras.layers.BatchNormalization()(encoder)\n        encoder = tf.keras.layers.Activation('swish')(encoder)\n\n        decoder = tf.keras.layers.Dropout(self.dropout_rates[1])(encoder)\n        decoder = tf.keras.layers.Dense(self.num_columns, name='decoder')(decoder)\n\n        x_reg = tf.keras.layers.Dense(self.hidden_units[1])(encoder)\n        x_reg = tf.keras.layers.BatchNormalization()(x_reg)\n        x_reg = tf.keras.layers.Activation('swish')(x_reg)\n        x_reg = tf.keras.layers.Dropout(self.dropout_rates[2])(x_reg)\n\n        out_reg = tf.keras.layers.Dense(1, activation='linear', name='target')(x_reg)\n\n        model = tf.keras.models.Model(inputs=inp, outputs=[decoder, out_reg])\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(learning_rate=self.lr),\n            loss={\"decoder\": tf.keras.losses.MeanSquaredError(),\n                  \"target\": tf.keras.losses.MeanSquaredError()},\n            loss_weights={\"decoder\": 0.3, \"target\": 1.0}\n        )\n        return model\n\n    def fit(self, X, y):\n        self.model.fit(\n            X, {\"decoder\": X, \"target\": y},\n            epochs=50,\n            batch_size=8192,\n            validation_split=0.2,\n            callbacks=[\n                tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),\n                tf.keras.callbacks.ReduceLROnPlateau(patience=5)\n            ],\n            verbose=0\n        )\n        return self\n\n    def predict(self, X):\n        _, y_pred = self.model.predict(X, verbose=0)\n        return y_pred.flatten()\n\ndef run_pipeline_with_stability_analysis(feature_compression: float, \n                                       label_compression: float,\n                                       use_adaptive: bool,\n                                       strategy_name: str,\n                                       n_stability_folds: int = 5):\n    \"\"\"\n    Run pipeline with stability analysis using multiple fold configurations\n    \"\"\"\n    print(f\"\\n{'='*80}\")\n    print(f\"Running pipeline with stability analysis: {strategy_name}\")\n    print(f\"Feature compression: {feature_compression:.2f}, Label compression: {label_compression:.2f}\")\n    print(f\"Adaptive: {use_adaptive}, Stability folds: {n_stability_folds}\")\n    print(f\"{'='*80}\\n\")\n    \n    # Load data\n    train = pd.read_parquet(CFG.train_path).reset_index(drop=True)\n    test = pd.read_parquet(CFG.test_path).reset_index(drop=True)\n    \n    # Define features\n    X_FEATURES = ['X363', 'X321', 'X405', 'X730', 'X523', 'X756', 'X589', 'X462', 'X779',\n                    'X25', 'X532', 'X520', 'X329', 'X383', 'X751', 'X535', 'X639', 'X596', 'X761',\n                \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n                \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X174\", \"X178\", \"X168\", \"X612\",\n                \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\"]\n    \n    selected_columns = X_FEATURES + [\"volume\"]\n    \n    # Select columns\n    train = train[selected_columns + [CFG.target]]\n    test = test[selected_columns]\n    \n    # Prepare data\n    X_train = train.drop(CFG.target, axis=1)\n    y_train = train[CFG.target]\n    X_test = test\n    \n    # Initialize and fit compressor\n    compressor = IntelligentAdaptiveCompressor()\n    \n    if use_adaptive:\n        compressor.fit(X_train, y_train, label_compression=label_compression, use_gradient_optimization=True)\n    else:\n        feature_overrides = {col: feature_compression for col in X_train.columns}\n        compressor.fit(X_train, y_train, \n                      feature_compression_override=feature_overrides,\n                      label_compression=label_compression,\n                      use_gradient_optimization=False)\n    \n    # Apply compression\n    X_train_compressed, y_train_compressed = compressor.transform(X_train, y_train)\n    X_test_compressed, _ = compressor.transform(X_test, None)\n    \n    # Save compressor\n    compressor.save(f'compressor_{strategy_name}.pkl')\n    \n    # Reduce memory\n    X_train_compressed = reduce_mem_usage(X_train_compressed, \"train\")\n    X_test_compressed = reduce_mem_usage(X_test_compressed, \"test\")\n    \n    # Run stability analysis with different fold configurations\n    stability_results = {}\n    all_test_predictions = []\n    \n    for fold_config in range(n_stability_folds):\n        print(f\"\\nStability fold configuration {fold_config + 1}/{n_stability_folds}\")\n        \n        # Create different fold splits\n        if fold_config == 0:\n            cv = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed)\n        elif fold_config == 1:\n            cv = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed + 1)\n        elif fold_config == 2:\n            cv = KFold(n_splits=8, shuffle=True, random_state=CFG.seed + 2)\n        elif fold_config == 3:\n            # Time series split for temporal stability\n            cv = TimeSeriesSplit(n_splits=min(5, CFG.n_folds))\n        else:\n            cv = KFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed + fold_config)\n        \n        # Initialize storage for this fold configuration\n        scores = {}\n        test_preds = {}\n        \n        # Train models\n        # LightGBM (gbdt)\n        lgbm_trainer = Trainer(\n            LGBMRegressor(**lgbm_params),\n            cv=cv,\n            metric=_pearsonr,\n            task=\"regression\",\n            metric_precision=6\n        )\n        lgbm_trainer.fit(X_train_compressed, y_train_compressed)\n        scores[\"LightGBM (gbdt)\"] = lgbm_trainer.fold_scores\n        test_preds[\"LightGBM (gbdt)\"] = lgbm_trainer.predict(X_test_compressed)\n        \n        # LightGBM (goss)\n        lgbm_goss_trainer = Trainer(\n            LGBMRegressor(**lgbm_goss_params),\n            cv=cv,\n            metric=_pearsonr,\n            task=\"regression\",\n            metric_precision=6\n        )\n        lgbm_goss_trainer.fit(X_train_compressed, y_train_compressed)\n        scores[\"LightGBM (goss)\"] = lgbm_goss_trainer.fold_scores\n        test_preds[\"LightGBM (goss)\"] = lgbm_goss_trainer.predict(X_test_compressed)\n        \n        # XGBoost\n        xgb_trainer = Trainer(\n            XGBRegressor(**xgb_params),\n            cv=cv,\n            metric=_pearsonr,\n            task=\"regression\",\n            metric_precision=6\n        )\n        xgb_trainer.fit(X_train_compressed, y_train_compressed)\n        scores[\"XGBoost\"] = xgb_trainer.fold_scores\n        test_preds[\"XGBoost\"] = xgb_trainer.predict(X_test_compressed)\n        \n        # Skip GANDALF and AutoEncoder for stability analysis (too slow)\n        # but calculate ensemble from the three main models\n        \n        # Calculate weighted predictions for this fold\n        ensemble_weights = {\n            \"LightGBM (gbdt)\": 0.4,\n            \"LightGBM (goss)\": 0.3,\n            \"XGBoost\": 0.3\n        }\n        \n        weighted_test_pred = np.zeros(len(X_test))\n        for model_name, weight in ensemble_weights.items():\n            weighted_test_pred += weight * test_preds[model_name]\n        \n        all_test_predictions.append(weighted_test_pred)\n        stability_results[f'fold_{fold_config}'] = scores\n    \n    # Calculate overall stability metrics\n    all_scores_flat = {}\n    for fold_results in stability_results.values():\n        for model, scores in fold_results.items():\n            if model not in all_scores_flat:\n                all_scores_flat[model] = []\n            all_scores_flat[model].extend(scores)\n    \n    stability_metrics = calculate_stability_metrics(all_scores_flat)\n    \n    # Average predictions across stability folds\n    final_predictions = np.mean(all_test_predictions, axis=0)\n    \n    # If labels were compressed, apply inverse transformation\n    if label_compression > 0 and compressor.label_profile is not None:\n        final_predictions = compressor.inverse_transform_predictions(final_predictions)\n    \n    # Clean up memory\n    del X_train, y_train, X_test, train, test\n    gc.collect()\n    \n    return final_predictions, stability_metrics, compressor, all_test_predictions\n\n# Dynamic weight optimization based on CV performance\ndef optimize_ensemble_weights(predictions_dict: Dict[str, np.ndarray], \n                            cv_scores: Dict[str, float]) -> Dict[str, float]:\n    \"\"\"Optimize ensemble weights based on CV performance and prediction diversity\"\"\"\n    \n    # Normalize CV scores\n    total_score = sum(cv_scores.values())\n    if total_score == 0:\n        return {k: 1/len(cv_scores) for k in cv_scores.keys()}\n    \n    base_weights = {k: v/total_score for k, v in cv_scores.items()}\n    \n    # Calculate prediction diversity (correlation between models)\n    correlations = {}\n    models = list(predictions_dict.keys())\n    \n    for i, model1 in enumerate(models):\n        for j, model2 in enumerate(models):\n            if i < j:\n                corr = np.corrcoef(predictions_dict[model1], predictions_dict[model2])[0, 1]\n                correlations[f\"{model1}_{model2}\"] = corr\n    \n    # Adjust weights based on diversity (less correlated models get higher weights)\n    diversity_bonus = {}\n    for model in models:\n        model_correlations = []\n        for corr_key, corr_val in correlations.items():\n            if model in corr_key:\n                model_correlations.append(corr_val)\n        \n        avg_correlation = np.mean(model_correlations) if model_correlations else 0.5\n        diversity_bonus[model] = 1 - avg_correlation  # Higher bonus for less correlated models\n    \n    # Combine base weights with diversity bonus\n    final_weights = {}\n    for model in models:\n        performance_weight = base_weights.get(model, 1/len(models))\n        diversity_weight = diversity_bonus.get(model, 0.5)\n        final_weights[model] = performance_weight * 0.7 + diversity_weight * 0.3\n    \n    # Normalize final weights\n    total_weight = sum(final_weights.values())\n    final_weights = {k: v/total_weight for k, v in final_weights.items()}\n    \n    return final_weights\n\n# Main execution\nif __name__ == \"__main__\":\n    # Extended compression grid with more intelligent strategies\n    compression_grid = [\n        # (feature_compression, label_compression, use_adaptive, name)\n        (0.0, 0.0, False, \"baseline_no_compression\"),\n        (0.2, 0.0, False, \"mild_features_20pct\"),\n        (0.4, 0.0, False, \"moderate_features_40pct\"),\n        (0.6, 0.0, False, \"strong_features_60pct\"),\n        (0.3, 0.05, False, \"balanced_30_5pct\"),\n        (0.5, 0.1, False, \"balanced_50_10pct\"),\n        (0.0, 0.0, True, \"adaptive_auto\"),\n        (0.0, 0.05, True, \"adaptive_label_5pct\"),\n        (0.0, 0.1, True, \"adaptive_label_10pct\"),\n        (0.0, 0.15, True, \"adaptive_label_15pct\"),\n    ]\n    \n    # Store results\n    all_results = {}\n    \n    # Run grid search with stability analysis\n    for feature_comp, label_comp, use_adaptive, name in compression_grid:\n        try:\n            predictions, stability_metrics, compressor, fold_predictions = \\\n                run_pipeline_with_stability_analysis(\n                    feature_comp, label_comp, use_adaptive, name, \n                    n_stability_folds=CFG.n_stability_folds\n                )\n            \n            all_results[name] = {\n                'predictions': predictions,\n                'stability_metrics': stability_metrics,\n                'compressor': compressor,\n                'fold_predictions': fold_predictions\n            }\n            \n            # Save individual submission\n            sub = pd.read_csv(CFG.sample_sub_path)\n            sub[\"prediction\"] = predictions\n            sub.to_csv(f\"submission_{name}.csv\", index=False)\n            print(f\"\\nSaved submission_{name}.csv\")\n            print(f\"Stability score: {stability_metrics.stability_score:.4f}\")\n            print(f\"Mean CV: {stability_metrics.mean_cv_score:.4f} ± {stability_metrics.std_cv_score:.4f}\")\n            \n        except Exception as e:\n            print(f\"\\nError in strategy {name}: {str(e)}\")\n            import traceback\n            traceback.print_exc()\n            continue\n    \n    # Select top 3 most stable strategies\n    print(\"\\n\" + \"=\"*80)\n    print(\"SELECTING TOP 3 MOST STABLE STRATEGIES\")\n    print(\"=\"*80)\n    \n    stability_scores = {\n        name: results['stability_metrics'].stability_score \n        for name, results in all_results.items()\n    }\n    \n    top_3_strategies = sorted(stability_scores.items(), key=lambda x: x[1], reverse=True)[:3]\n    \n    print(\"\\nTop 3 most stable strategies:\")\n    for i, (strategy, score) in enumerate(top_3_strategies, 1):\n        metrics = all_results[strategy]['stability_metrics']\n        print(f\"{i}. {strategy}: stability={score:.4f}, \"\n              f\"mean={metrics.mean_cv_score:.4f}, std={metrics.std_cv_score:.4f}\")\n    \n    # Create final ensemble from top 3 strategies\n    print(\"\\nCreating final ensemble from top 3 strategies...\")\n    \n    # Calculate dynamic weights based on stability and performance\n    top_predictions = {}\n    top_cv_scores = {}\n    \n    for strategy, _ in top_3_strategies:\n        top_predictions[strategy] = all_results[strategy]['predictions']\n        top_cv_scores[strategy] = all_results[strategy]['stability_metrics'].mean_cv_score\n    \n    # Optimize weights\n    optimal_weights = optimize_ensemble_weights(top_predictions, top_cv_scores)\n    \n    print(\"\\nOptimal ensemble weights:\")\n    for strategy, weight in optimal_weights.items():\n        print(f\"  {strategy}: {weight:.3f}\")\n    \n    # Create final ensemble\n    final_ensemble = np.zeros_like(list(top_predictions.values())[0])\n    for strategy, weight in optimal_weights.items():\n        final_ensemble += weight * top_predictions[strategy]\n    \n    # Save final ensemble\n    sub = pd.read_csv(CFG.sample_sub_path)\n    sub[\"prediction\"] = final_ensemble\n    sub.to_csv(\"submission_final_stable_ensemble.csv\", index=False)\n    print(\"\\nSaved submission_final_stable_ensemble.csv\")\n    \n    # Create comprehensive visualization\n    # 1. Stability comparison plot\n    plt.figure(figsize=(12, 8))\n    strategies = list(stability_scores.keys())\n    scores = list(stability_scores.values())\n    means = [all_results[s]['stability_metrics'].mean_cv_score for s in strategies]\n    stds = [all_results[s]['stability_metrics'].std_cv_score for s in strategies]\n    \n    x = np.arange(len(strategies))\n    plt.bar(x, scores, alpha=0.7, label='Stability Score')\n    plt.errorbar(x, means, yerr=stds, fmt='o', color='red', label='Mean ± Std')\n    \n    plt.xticks(x, strategies, rotation=45, ha='right')\n    plt.ylabel('Score')\n    plt.title('Strategy Stability Analysis')\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    plt.tight_layout()\n    plt.savefig('stability_analysis.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    # 2. Prediction correlation heatmap for top strategies\n    plt.figure(figsize=(8, 6))\n    corr_matrix = np.corrcoef([top_predictions[s] for s, _ in top_3_strategies])\n    sns.heatmap(corr_matrix, \n                xticklabels=[s for s, _ in top_3_strategies],\n                yticklabels=[s for s, _ in top_3_strategies],\n                annot=True, fmt='.3f', cmap='coolwarm', center=0.5)\n    plt.title('Top 3 Strategies Prediction Correlation')\n    plt.tight_layout()\n    plt.savefig('top_strategies_correlation.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    # 3. Feature compression analysis for adaptive strategies\n    adaptive_strategies = [name for _, _, use_adaptive, name in compression_grid if use_adaptive]\n    if adaptive_strategies and any(s in all_results for s in adaptive_strategies):\n        # Get one adaptive compressor for analysis\n        adaptive_name = next(s for s in adaptive_strategies if s in all_results)\n        compressor = all_results[adaptive_name]['compressor']\n        \n        # Extract compression strengths\n        feature_data = []\n        for feat, profile in list(compressor.compression_profiles.items())[:30]:  # Top 30 features\n            feature_data.append({\n                'Feature': feat,\n                'Compression': profile.strength,\n                'Noise': profile.noise_level,\n                'MI Score': profile.mutual_info_score,\n                'Gradient': profile.gradient_sensitivity\n            })\n        \n        feature_df = pd.DataFrame(feature_data)\n        feature_df = feature_df.sort_values('Compression', ascending=False)\n        \n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n        \n        # Compression strength by feature\n        ax1.barh(feature_df['Feature'], feature_df['Compression'])\n        ax1.set_xlabel('Compression Strength')\n        ax1.set_title('Adaptive Compression by Feature')\n        ax1.grid(True, alpha=0.3)\n        \n        # Scatter plot: Noise vs MI Score\n        scatter = ax2.scatter(feature_df['Noise'], feature_df['MI Score'], \n                            c=feature_df['Compression'], cmap='viridis', s=100)\n        ax2.set_xlabel('Noise Level')\n        ax2.set_ylabel('Mutual Information Score')\n        ax2.set_title('Feature Analysis: Noise vs Information')\n        plt.colorbar(scatter, ax=ax2, label='Compression')\n        ax2.grid(True, alpha=0.3)\n        \n        plt.tight_layout()\n        plt.savefig('adaptive_compression_analysis.png', dpi=300, bbox_inches='tight')\n        plt.show()\n    \n    # Save comprehensive summary\n    summary_data = []\n    for name, results in all_results.items():\n        metrics = results['stability_metrics']\n        summary_data.append({\n            'Strategy': name,\n            'Mean_CV': metrics.mean_cv_score,\n            'Std_CV': metrics.std_cv_score,\n            'Stability_Score': metrics.stability_score,\n            'Inter_Fold_Correlation': metrics.inter_fold_correlation,\n            'Is_Top_3': name in [s for s, _ in top_3_strategies],\n            'Final_Weight': optimal_weights.get(name, 0)\n        })\n    \n    summary_df = pd.DataFrame(summary_data)\n    summary_df = summary_df.sort_values('Stability_Score', ascending=False)\n    summary_df.to_csv('intelligent_compression_summary.csv', index=False)\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"INTELLIGENT COMPRESSION PIPELINE COMPLETED!\")\n    print(\"=\"*80)\n    print(\"Generated files:\")\n    print(\"- Individual submissions for each strategy\")\n    print(\"- submission_final_stable_ensemble.csv (TOP 3 ENSEMBLE)\")\n    print(\"- Compression profiles (.pkl files)\")\n    print(\"- stability_analysis.png\")\n    print(\"- top_strategies_correlation.png\")\n    print(\"- adaptive_compression_analysis.png\")\n    print(\"- intelligent_compression_summary.csv\")\n    print(\"=\"*80)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}