{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Educational Version: Intelligent Adaptive Compressor Deep Dive\n# This version focuses on understanding how the compressor works with detailed visualizations\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import pearsonr, kurtosis, skew\nfrom sklearn.metrics import mutual_info_score\nfrom sklearn.ensemble import RandomForestRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import KFold\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set style for beautiful visualizations\nplt.style.use('seaborn-v0_8-darkgrid')\nsns.set_palette(\"husl\")\n\nprint(\"=\"*80)\nprint(\"INTELLIGENT ADAPTIVE COMPRESSOR - EDUCATIONAL DEEP DIVE\")\nprint(\"=\"*80)\nprint(\"\\nThis notebook demonstrates how intelligent compression works by:\")\nprint(\"1. Analyzing feature characteristics\")\nprint(\"2. Selecting optimal compression strategies\")\nprint(\"3. Visualizing the transformation process\")\nprint(\"4. Measuring impact on model performance\")\nprint(\"=\"*80)\n\n# Configuration for educational version\nclass Config:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    n_samples = 10000  # Focus on recent data\n    n_features = 30    # Manageable number for visualization\n    seed = 42\n    \n# Load data with focus on most recent samples\nprint(\"\\n📊 LOADING DATA...\")\ntrain_df = pd.read_parquet(Config.train_path)\nprint(f\"Full dataset shape: {train_df.shape}\")\n\n# Take most recent samples\ntrain_df = train_df.tail(Config.n_samples).reset_index(drop=True)\nprint(f\"Using most recent {Config.n_samples} samples\")\n\n# Select diverse features for demonstration\nselected_features = [\n    # Market microstructure\n    'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume',\n    # Various X features with different characteristics\n    'X287', 'X446', 'X66', 'X123', 'X385', 'X25', 'X3', \n    'X415', 'X345', 'X37', 'X174', 'X298', 'X178', 'X168',\n    'X598', 'X603', 'X674', 'X756', 'X589', 'X462', 'X779',\n    'X532', 'X520', 'X329', 'X383'\n]\n\n# Ensure we have the features\navailable_features = [f for f in selected_features if f in train_df.columns]\nX_train = train_df[available_features]\ny_train = train_df['label']\n\nprint(f\"\\n✅ Selected {len(available_features)} features for analysis\")\nprint(f\"Features: {', '.join(available_features[:10])}...\")\n\n# Educational Compressor with detailed logging\nclass EducationalIntelligentCompressor:\n    \"\"\"Educational version with extensive visualization and explanation\"\"\"\n    \n    def __init__(self):\n        self.compression_profiles = {}\n        self.analysis_results = {}\n        self.feature_importance = {}\n        \n    def analyze_feature_educational(self, data, target, feature_name):\n        \"\"\"Analyze feature with detailed explanations\"\"\"\n        print(f\"\\n{'='*60}\")\n        print(f\"🔍 ANALYZING FEATURE: {feature_name}\")\n        print(f\"{'='*60}\")\n        \n        # Remove NaN values\n        mask = ~(np.isnan(data) | np.isnan(target))\n        clean_data = data[mask]\n        clean_target = target[mask]\n        \n        if len(clean_data) < 10:\n            print(\"❌ Too few valid samples for analysis\")\n            return None\n            \n        # Basic statistics\n        median = np.median(clean_data)\n        mad = np.median(np.abs(clean_data - median))\n        mean = np.mean(clean_data)\n        std = np.std(clean_data)\n        \n        print(f\"\\n📈 Basic Statistics:\")\n        print(f\"   Mean: {mean:.4f}\")\n        print(f\"   Median: {median:.4f}\")\n        print(f\"   MAD: {mad:.4f}\")\n        print(f\"   Std Dev: {std:.4f}\")\n        \n        # Create figure for this feature\n        fig = plt.figure(figsize=(20, 12))\n        \n        # 1. Distribution Plot\n        ax1 = plt.subplot(3, 3, 1)\n        plt.hist(clean_data, bins=50, alpha=0.7, color='skyblue', edgecolor='black')\n        plt.axvline(median, color='red', linestyle='--', label=f'Median: {median:.2f}')\n        plt.axvline(mean, color='green', linestyle='--', label=f'Mean: {mean:.2f}')\n        plt.title(f'{feature_name} - Distribution')\n        plt.xlabel('Value')\n        plt.ylabel('Frequency')\n        plt.legend()\n        \n        # 2. Box Plot for Outliers\n        ax2 = plt.subplot(3, 3, 2)\n        box_plot = plt.boxplot(clean_data, vert=True, patch_artist=True)\n        box_plot['boxes'][0].set_facecolor('lightcoral')\n        plt.title(f'{feature_name} - Outlier Detection')\n        plt.ylabel('Value')\n        \n        # Calculate outlier metrics\n        if mad > 0:\n            z_scores = np.abs((clean_data - median) / mad)\n            outlier_score = np.mean(z_scores > 3)\n            extreme_outlier_score = np.mean(z_scores > 6)\n            \n            print(f\"\\n🎯 Outlier Analysis:\")\n            print(f\"   Outliers (>3 MAD): {outlier_score*100:.2f}%\")\n            print(f\"   Extreme outliers (>6 MAD): {extreme_outlier_score*100:.2f}%\")\n        else:\n            outlier_score = 0\n            extreme_outlier_score = 0\n            \n        # 3. Feature vs Target Scatter\n        ax3 = plt.subplot(3, 3, 3)\n        scatter = plt.scatter(clean_data, clean_target, alpha=0.5, s=10)\n        plt.title(f'{feature_name} vs Target')\n        plt.xlabel(feature_name)\n        plt.ylabel('Target')\n        \n        # Add correlation\n        corr = pearsonr(clean_data, clean_target)[0]\n        plt.text(0.05, 0.95, f'Correlation: {corr:.3f}', \n                transform=ax3.transAxes, verticalalignment='top',\n                bbox=dict(boxstyle='round', facecolor='wheat', alpha=0.5))\n        \n        # 4. Noise Analysis (FFT)\n        ax4 = plt.subplot(3, 3, 4)\n        if len(clean_data) > 100:\n            fft = np.fft.fft(clean_data)\n            frequencies = np.fft.fftfreq(len(clean_data))\n            magnitude = np.abs(fft)\n            \n            # Plot only positive frequencies\n            pos_mask = frequencies > 0\n            plt.plot(frequencies[pos_mask], magnitude[pos_mask])\n            plt.title(f'{feature_name} - Frequency Spectrum (Noise Analysis)')\n            plt.xlabel('Frequency')\n            plt.ylabel('Magnitude')\n            plt.yscale('log')\n            \n            # Calculate noise level\n            noise_level = np.sum(magnitude[len(magnitude)//2:]) / np.sum(magnitude)\n            print(f\"\\n🔊 Noise Analysis:\")\n            print(f\"   Noise level (high-freq ratio): {noise_level:.3f}\")\n        else:\n            noise_level = 0.5\n            \n        # 5. Distribution Characteristics\n        ax5 = plt.subplot(3, 3, 5)\n        try:\n            kurt = kurtosis(clean_data)\n            skewness = skew(clean_data)\n            \n            # Q-Q plot for normality\n            from scipy import stats\n            stats.probplot(clean_data, dist=\"norm\", plot=plt)\n            plt.title(f'{feature_name} - Q-Q Plot (Normality Test)')\n            \n            print(f\"\\n📊 Distribution Characteristics:\")\n            print(f\"   Kurtosis: {kurt:.3f} (0=normal, >0=heavy tails)\")\n            print(f\"   Skewness: {skewness:.3f} (0=symmetric)\")\n            \n            distribution_score = 1 / (1 + np.exp(-(kurt / 10 + abs(skewness) / 5)))\n        except:\n            distribution_score = 0.5\n            \n        # 6. Gradient Sensitivity\n        ax6 = plt.subplot(3, 3, 6)\n        # Sort by feature value\n        sorted_idx = np.argsort(clean_data)\n        sorted_feature = clean_data[sorted_idx]\n        sorted_target = clean_target[sorted_idx]\n        \n        # Calculate local gradients\n        window = 50  # Window for gradient calculation\n        gradients = []\n        positions = []\n        \n        for i in range(window, len(sorted_feature) - window):\n            local_feature = sorted_feature[i-window:i+window]\n            local_target = sorted_target[i-window:i+window]\n            \n            if np.std(local_feature) > 0:\n                # Local linear regression\n                coef = np.polyfit(local_feature, local_target, 1)[0]\n                gradients.append(abs(coef))\n                positions.append(sorted_feature[i])\n        \n        if gradients:\n            plt.plot(positions, gradients, alpha=0.7)\n            plt.title(f'{feature_name} - Local Gradient Sensitivity')\n            plt.xlabel(feature_name)\n            plt.ylabel('|dTarget/dFeature|')\n            \n            gradient_sensitivity = np.percentile(gradients, 90)\n            print(f\"\\n📐 Gradient Sensitivity:\")\n            print(f\"   90th percentile gradient: {gradient_sensitivity:.3f}\")\n            print(f\"   Interpretation: {'High' if gradient_sensitivity > 5 else 'Low'} sensitivity\")\n        else:\n            gradient_sensitivity = 0\n            \n        # 7. Mutual Information\n        ax7 = plt.subplot(3, 3, 7)\n        try:\n            # Discretize for MI calculation\n            n_bins = min(20, len(clean_data) // 10)\n            feature_bins = pd.qcut(clean_data, n_bins, labels=False, duplicates='drop')\n            target_bins = pd.qcut(clean_target, n_bins, labels=False, duplicates='drop')\n            \n            # Create 2D histogram for visualization\n            plt.hist2d(clean_data, clean_target, bins=20, cmap='YlOrRd')\n            plt.colorbar(label='Count')\n            plt.title(f'{feature_name} - Joint Distribution')\n            plt.xlabel(feature_name)\n            plt.ylabel('Target')\n            \n            # Calculate MI\n            mutual_info = mutual_info_score(feature_bins, target_bins)\n            max_entropy = np.log2(n_bins)\n            mutual_info_score_norm = mutual_info / max_entropy if max_entropy > 0 else 0\n            \n            print(f\"\\n🎲 Information Theory:\")\n            print(f\"   Mutual Information: {mutual_info:.3f}\")\n            print(f\"   Normalized MI: {mutual_info_score_norm:.3f}\")\n            print(f\"   Interpretation: Feature is {'highly' if mutual_info_score_norm > 0.5 else 'weakly'} informative\")\n        except:\n            mutual_info_score_norm = 0\n            \n        # 8. Compression Decision Visualization\n        ax8 = plt.subplot(3, 3, 8)\n        compression_factors = {\n            'Noise': noise_level * 0.3,\n            'Outliers': outlier_score * 0.2 + extreme_outlier_score * 0.1,\n            'Distribution': distribution_score * 0.2,\n            'Gradient': (1 - np.tanh(gradient_sensitivity / 10)) * 0.1,\n            'Mutual Info': (1 - mutual_info_score_norm) * 0.3\n        }\n        \n        # Bar plot of factors\n        factors = list(compression_factors.keys())\n        values = list(compression_factors.values())\n        bars = plt.bar(factors, values, color=['red', 'orange', 'yellow', 'green', 'blue'])\n        plt.title('Compression Factor Breakdown')\n        plt.ylabel('Contribution to Compression')\n        plt.xticks(rotation=45)\n        \n        # Add values on bars\n        for bar, value in zip(bars, values):\n            plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01,\n                    f'{value:.3f}', ha='center', va='bottom')\n        \n        optimal_compression = np.clip(sum(compression_factors.values()), 0, 0.8)\n        \n        # 9. Compression Method Selection\n        ax9 = plt.subplot(3, 3, 9)\n        \n        # Decision tree visualization\n        methods = ['Robust Tanh', 'Soft Clip', 'Sigmoid', 'Log']\n        scores = []\n        \n        if outlier_score > 0.15 or distribution_score > 0.7:\n            method = 'robust_tanh'\n            scores = [1.0, 0.3, 0.2, 0.1]\n        elif gradient_sensitivity > 5:\n            method = 'soft_clip'\n            scores = [0.2, 1.0, 0.3, 0.1]\n        elif abs(skewness) > 2:\n            method = 'log'\n            scores = [0.1, 0.2, 0.3, 1.0]\n        else:\n            method = 'sigmoid'\n            scores = [0.2, 0.3, 1.0, 0.1]\n            \n        bars = plt.bar(methods, scores, color=['#ff9999', '#66b3ff', '#99ff99', '#ffcc99'])\n        plt.title('Compression Method Selection')\n        plt.ylabel('Suitability Score')\n        plt.xticks(rotation=45)\n        \n        # Highlight selected method\n        selected_idx = methods.index(method.replace('_', ' ').title())\n        bars[selected_idx].set_edgecolor('black')\n        bars[selected_idx].set_linewidth(3)\n        \n        plt.tight_layout()\n        plt.savefig(f'feature_analysis_{feature_name}.png', dpi=150, bbox_inches='tight')\n        plt.show()\n        \n        print(f\"\\n🎯 COMPRESSION DECISION:\")\n        print(f\"   Optimal compression strength: {optimal_compression:.3f}\")\n        print(f\"   Selected method: {method}\")\n        print(f\"   Reasoning: \", end=\"\")\n        if method == 'robust_tanh':\n            print(\"High outliers or non-normal distribution\")\n        elif method == 'soft_clip':\n            print(\"High gradient sensitivity - gentle compression needed\")\n        elif method == 'log':\n            print(\"Highly skewed distribution\")\n        else:\n            print(\"Standard compression for normal-like distribution\")\n            \n        return {\n            'median': median,\n            'mad': mad,\n            'noise_level': noise_level,\n            'outlier_score': outlier_score,\n            'distribution_score': distribution_score,\n            'gradient_sensitivity': gradient_sensitivity,\n            'mutual_info_score': mutual_info_score_norm,\n            'optimal_compression': optimal_compression,\n            'method': method,\n            'compression_factors': compression_factors\n        }\n    \n    def visualize_compression_effect(self, original_data, compressed_data, feature_name, profile):\n        \"\"\"Visualize the before/after of compression\"\"\"\n        fig, axes = plt.subplots(2, 3, figsize=(18, 10))\n        fig.suptitle(f'Compression Effect on {feature_name}', fontsize=16)\n        \n        # 1. Distribution comparison\n        ax = axes[0, 0]\n        ax.hist(original_data, bins=50, alpha=0.5, label='Original', color='blue', density=True)\n        ax.hist(compressed_data, bins=50, alpha=0.5, label='Compressed', color='red', density=True)\n        ax.set_title('Distribution Comparison')\n        ax.set_xlabel('Value')\n        ax.set_ylabel('Density')\n        ax.legend()\n        \n        # 2. Scatter plot: Original vs Compressed\n        ax = axes[0, 1]\n        ax.scatter(original_data, compressed_data, alpha=0.5, s=10)\n        ax.plot([original_data.min(), original_data.max()], \n                [original_data.min(), original_data.max()], \n                'r--', label='y=x (no compression)')\n        ax.set_title('Compression Transformation')\n        ax.set_xlabel('Original Value')\n        ax.set_ylabel('Compressed Value')\n        ax.legend()\n        \n        # 3. Compression function visualization\n        ax = axes[0, 2]\n        sorted_orig = np.sort(original_data)\n        sorted_comp = compressed_data[np.argsort(original_data)]\n        ax.plot(sorted_orig, sorted_comp, linewidth=2)\n        ax.set_title(f'Compression Function ({profile[\"method\"]})')\n        ax.set_xlabel('Original Value')\n        ax.set_ylabel('Compressed Value')\n        ax.grid(True, alpha=0.3)\n        \n        # 4. Outlier handling\n        ax = axes[1, 0]\n        outlier_threshold = np.percentile(np.abs(original_data - np.median(original_data)), 95)\n        outliers_mask = np.abs(original_data - np.median(original_data)) > outlier_threshold\n        \n        ax.scatter(original_data[~outliers_mask], compressed_data[~outliers_mask], \n                  alpha=0.5, s=10, label='Normal values')\n        ax.scatter(original_data[outliers_mask], compressed_data[outliers_mask], \n                  alpha=0.8, s=30, color='red', label='Outliers')\n        ax.set_title('Outlier Compression')\n        ax.set_xlabel('Original Value')\n        ax.set_ylabel('Compressed Value')\n        ax.legend()\n        \n        # 5. Statistics comparison\n        ax = axes[1, 1]\n        stats_names = ['Mean', 'Std', 'Min', 'Max', 'Range']\n        original_stats = [\n            np.mean(original_data),\n            np.std(original_data),\n            np.min(original_data),\n            np.max(original_data),\n            np.max(original_data) - np.min(original_data)\n        ]\n        compressed_stats = [\n            np.mean(compressed_data),\n            np.std(compressed_data),\n            np.min(compressed_data),\n            np.max(compressed_data),\n            np.max(compressed_data) - np.min(compressed_data)\n        ]\n        \n        x = np.arange(len(stats_names))\n        width = 0.35\n        ax.bar(x - width/2, original_stats, width, label='Original', alpha=0.8)\n        ax.bar(x + width/2, compressed_stats, width, label='Compressed', alpha=0.8)\n        ax.set_title('Statistical Comparison')\n        ax.set_xticks(x)\n        ax.set_xticklabels(stats_names)\n        ax.legend()\n        \n        # 6. Compression strength visualization\n        ax = axes[1, 2]\n        # Show how much each value was compressed\n        compression_amount = np.abs(compressed_data - original_data)\n        scatter = ax.scatter(original_data, compression_amount, \n                           c=np.abs(original_data - np.median(original_data)),\n                           cmap='viridis', alpha=0.6, s=10)\n        ax.set_title('Compression Amount by Value')\n        ax.set_xlabel('Original Value')\n        ax.set_ylabel('|Original - Compressed|')\n        plt.colorbar(scatter, ax=ax, label='Distance from Median')\n        \n        plt.tight_layout()\n        plt.savefig(f'compression_effect_{feature_name}.png', dpi=150, bbox_inches='tight')\n        plt.show()\n        \n        # Print compression statistics\n        print(f\"\\n📊 Compression Statistics for {feature_name}:\")\n        print(f\"   Original range: [{original_data.min():.3f}, {original_data.max():.3f}]\")\n        print(f\"   Compressed range: [{compressed_data.min():.3f}, {compressed_data.max():.3f}]\")\n        print(f\"   Range reduction: {(1 - (compressed_data.max() - compressed_data.min()) / (original_data.max() - original_data.min() + 1e-10)) * 100:.1f}%\")\n        print(f\"   Std reduction: {(1 - np.std(compressed_data) / (np.std(original_data) + 1e-10)) * 100:.1f}%\")\n        print(f\"   Outliers tamed: {np.sum(outliers_mask)} values\")\n    \n    def fit(self, X, y):\n        \"\"\"Fit compressor with educational output\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"🚀 FITTING INTELLIGENT ADAPTIVE COMPRESSOR\")\n        print(\"=\"*80)\n        \n        # Calculate feature importance\n        print(\"\\n📊 Calculating feature importance using Random Forest...\")\n        rf = RandomForestRegressor(n_estimators=50, random_state=42, n_jobs=-1)\n        rf.fit(X, y)\n        self.feature_importance = dict(zip(X.columns, rf.feature_importances_))\n        \n        # Visualize feature importance\n        plt.figure(figsize=(12, 6))\n        importance_df = pd.DataFrame({\n            'Feature': list(self.feature_importance.keys()),\n            'Importance': list(self.feature_importance.values())\n        }).sort_values('Importance', ascending=False).head(20)\n        \n        plt.bar(importance_df['Feature'], importance_df['Importance'])\n        plt.xticks(rotation=45, ha='right')\n        plt.title('Top 20 Feature Importances')\n        plt.xlabel('Feature')\n        plt.ylabel('Importance Score')\n        plt.tight_layout()\n        plt.savefig('feature_importance.png', dpi=150, bbox_inches='tight')\n        plt.show()\n        \n        # Analyze each feature\n        for i, col in enumerate(X.columns):\n            if i >= 5:  # Limit detailed analysis to first 5 features for brevity\n                print(f\"\\n⏭️  Analyzing remaining features in batch mode...\")\n                # Quick analysis for remaining features\n                for remaining_col in X.columns[5:]:\n                    analysis = self.analyze_feature_educational(\n                        X[remaining_col].values, \n                        y.values, \n                        remaining_col\n                    )\n                    if analysis:\n                        self.analysis_results[remaining_col] = analysis\n                break\n            else:\n                analysis = self.analyze_feature_educational(\n                    X[col].values, \n                    y.values, \n                    col\n                )\n                if analysis:\n                    self.analysis_results[col] = analysis\n        \n        # Summary visualization\n        self._create_summary_visualization()\n        \n        return self\n    \n    def _create_summary_visualization(self):\n        \"\"\"Create summary of all features' compression profiles\"\"\"\n        if not self.analysis_results:\n            return\n            \n        # Create summary DataFrame\n        summary_data = []\n        for feature, analysis in self.analysis_results.items():\n            summary_data.append({\n                'Feature': feature,\n                'Compression': analysis['optimal_compression'],\n                'Method': analysis['method'],\n                'Noise': analysis['noise_level'],\n                'Outliers': analysis['outlier_score'],\n                'MI Score': analysis['mutual_info_score'],\n                'Gradient': analysis['gradient_sensitivity']\n            })\n        \n        summary_df = pd.DataFrame(summary_data)\n        \n        # Create comprehensive summary plot\n        fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n        fig.suptitle('Compression Analysis Summary', fontsize=16)\n        \n        # 1. Compression strength by feature\n        ax = axes[0, 0]\n        summary_df_sorted = summary_df.sort_values('Compression', ascending=False).head(15)\n        bars = ax.bar(summary_df_sorted['Feature'], summary_df_sorted['Compression'])\n        \n        # Color by method\n        method_colors = {\n            'robust_tanh': '#ff9999',\n            'soft_clip': '#66b3ff',\n            'sigmoid': '#99ff99',\n            'log': '#ffcc99'\n        }\n        for bar, method in zip(bars, summary_df_sorted['Method']):\n            bar.set_color(method_colors.get(method, 'gray'))\n        \n        ax.set_title('Compression Strength by Feature')\n        ax.set_xlabel('Feature')\n        ax.set_ylabel('Compression Strength')\n        ax.set_xticklabels(summary_df_sorted['Feature'], rotation=45, ha='right')\n        \n        # 2. Method distribution\n        ax = axes[0, 1]\n        method_counts = summary_df['Method'].value_counts()\n        colors = [method_colors.get(m, 'gray') for m in method_counts.index]\n        ax.pie(method_counts.values, labels=method_counts.index, autopct='%1.1f%%', colors=colors)\n        ax.set_title('Distribution of Compression Methods')\n        \n        # 3. Noise vs MI Score scatter\n        ax = axes[1, 0]\n        scatter = ax.scatter(summary_df['Noise'], summary_df['MI Score'], \n                           c=summary_df['Compression'], s=100, cmap='viridis', alpha=0.6)\n        ax.set_xlabel('Noise Level')\n        ax.set_ylabel('Mutual Information Score')\n        ax.set_title('Feature Quality: Noise vs Information')\n        plt.colorbar(scatter, ax=ax, label='Compression Strength')\n        \n        # Add feature labels for interesting points\n        for idx, row in summary_df.iterrows():\n            if row['Noise'] > 0.7 or row['MI Score'] > 0.5:\n                ax.annotate(row['Feature'], (row['Noise'], row['MI Score']), \n                          fontsize=8, alpha=0.7)\n        \n        # 4. Compression factors breakdown\n        ax = axes[1, 1]\n        # Average contribution of each factor\n        factor_contributions = {\n            'Noise': [],\n            'Outliers': [],\n            'Distribution': [],\n            'Gradient': [],\n            'Mutual Info': []\n        }\n        \n        for feature, analysis in self.analysis_results.items():\n            factors = analysis['compression_factors']\n            for key in factor_contributions:\n                if key in factors:\n                    factor_contributions[key].append(factors[key])\n        \n        avg_contributions = {k: np.mean(v) for k, v in factor_contributions.items()}\n        \n        ax.bar(avg_contributions.keys(), avg_contributions.values(), \n               color=['red', 'orange', 'yellow', 'green', 'blue'])\n        ax.set_title('Average Compression Factor Contributions')\n        ax.set_ylabel('Average Contribution')\n        ax.set_xticklabels(avg_contributions.keys(), rotation=45)\n        \n        plt.tight_layout()\n        plt.savefig('compression_summary.png', dpi=150, bbox_inches='tight')\n        plt.show()\n        \n        # Print insights\n        print(\"\\n\" + \"=\"*80)\n        print(\"🎯 KEY INSIGHTS FROM COMPRESSION ANALYSIS\")\n        print(\"=\"*80)\n        \n        print(f\"\\n1. Method Distribution:\")\n        for method, count in method_counts.items():\n            print(f\"   {method}: {count} features ({count/len(summary_df)*100:.1f}%)\")\n        \n        print(f\"\\n2. Compression Strength:\")\n        print(f\"   Average: {summary_df['Compression'].mean():.3f}\")\n        print(f\"   Range: [{summary_df['Compression'].min():.3f}, {summary_df['Compression'].max():.3f}]\")\n        \n        print(f\"\\n3. Feature Quality:\")\n        high_quality = summary_df[(summary_df['MI Score'] > 0.3) & (summary_df['Noise'] < 0.5)]\n        print(f\"   High quality features (high MI, low noise): {len(high_quality)}\")\n        if len(high_quality) > 0:\n            print(f\"   Examples: {', '.join(high_quality['Feature'].head(5).tolist())}\")\n        \n        print(f\"\\n4. Outlier-heavy Features:\")\n        outlier_features = summary_df[summary_df['Outliers'] > 0.1]\n        print(f\"   Features with >10% outliers: {len(outlier_features)}\")\n        if len(outlier_features) > 0:\n            print(f\"   Examples: {', '.join(outlier_features['Feature'].head(5).tolist())}\")\n    \n    def transform(self, X):\n        \"\"\"Transform features with visualization\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"🔄 APPLYING COMPRESSION TRANSFORMATIONS\")\n        print(\"=\"*80)\n        \n        X_compressed = X.copy()\n        \n        # Transform first few features with detailed visualization\n        for i, col in enumerate(X.columns):\n            if col in self.analysis_results:\n                profile = self.analysis_results[col]\n                original_data = X[col].values\n                compressed_data = self._compress_data(original_data, profile)\n                X_compressed[col] = compressed_data\n                \n                # Detailed visualization for first 3 features\n                if i < 3:\n                    self.visualize_compression_effect(\n                        original_data, \n                        compressed_data, \n                        col, \n                        profile\n                    )\n        \n        return X_compressed\n    \n    def _compress_data(self, data, profile):\n        \"\"\"Apply compression using the selected method\"\"\"\n        # Handle NaN values\n        nan_mask = np.isnan(data)\n        data_clean = data[~nan_mask]\n        \n        if len(data_clean) == 0 or profile['optimal_compression'] == 0:\n            return data\n        \n        # Normalize using MAD\n        if profile['mad'] > 0:\n            normalized = (data - profile['median']) / (profile['mad'] * 6)\n        else:\n            return data\n        \n        strength = profile['optimal_compression']\n        method = profile['method']\n        \n        # Apply compression method\n        if method == 'robust_tanh':\n            alpha = 1 - strength\n            compressed = np.where(\n                np.abs(normalized) < 3,\n                np.tanh(normalized * alpha),\n                np.sign(normalized) * (0.995 + 0.005 * np.tanh((np.abs(normalized) - 3) * alpha))\n            )\n        elif method == 'soft_clip':\n            threshold = 2 * (1 - strength + 0.1)\n            compressed = np.where(\n                np.abs(normalized) < threshold,\n                normalized,\n                np.sign(normalized) * (threshold + np.log1p(np.abs(normalized) - threshold) * 0.5)\n            )\n        elif method == 'sigmoid':\n            compressed = 2 / (1 + np.exp(-normalized * (1 - strength) * 2)) - 1\n        elif method == 'log':\n            sign = np.sign(normalized)\n            abs_norm = np.abs(normalized)\n            compressed = sign * np.log1p(abs_norm * (1 - strength)) / np.log1p(1 - strength + 1e-10)\n        else:\n            compressed = normalized\n        \n        # Scale back\n        result = compressed * (profile['mad'] * 6) + profile['median']\n        \n        # Preserve NaN values\n        if nan_mask.any():\n            full_result = np.full_like(data, np.nan, dtype=np.float64)\n            full_result[~nan_mask] = result[~nan_mask]\n            return full_result\n        \n        return result\n\ndef evaluate_compression_impact(X_original, X_compressed, y, feature_names):\n    \"\"\"Evaluate the impact of compression on model performance\"\"\"\n    print(\"\\n\" + \"=\"*80)\n    print(\"📈 EVALUATING COMPRESSION IMPACT ON MODEL PERFORMANCE\")\n    print(\"=\"*80)\n    \n    # Train models on both original and compressed data\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    \n    # LightGBM parameters\n    lgb_params = {\n        'n_estimators': 100,\n        'learning_rate': 0.1,\n        'num_leaves': 31,\n        'random_state': 42,\n        'verbose': -1\n    }\n    \n    original_scores = []\n    compressed_scores = []\n    \n    print(\"\\n🔄 Running 5-fold cross-validation...\")\n    \n    for fold, (train_idx, val_idx) in enumerate(kf.split(X_original)):\n        # Original data\n        model_orig = LGBMRegressor(**lgb_params)\n        model_orig.fit(X_original.iloc[train_idx], y.iloc[train_idx])\n        pred_orig = model_orig.predict(X_original.iloc[val_idx])\n        score_orig = pearsonr(y.iloc[val_idx], pred_orig)[0]\n        original_scores.append(score_orig)\n        \n        # Compressed data\n        model_comp = LGBMRegressor(**lgb_params)\n        model_comp.fit(X_compressed.iloc[train_idx], y.iloc[train_idx])\n        pred_comp = model_comp.predict(X_compressed.iloc[val_idx])\n        score_comp = pearsonr(y.iloc[val_idx], pred_comp)[0]\n        compressed_scores.append(score_comp)\n        \n        print(f\"   Fold {fold+1}: Original={score_orig:.4f}, Compressed={score_comp:.4f}\")\n    \n    # Results visualization\n    fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n    \n    # 1. Score comparison\n    ax = axes[0]\n    x = range(len(original_scores))\n    width = 0.35\n    ax.bar([i - width/2 for i in x], original_scores, width, label='Original', alpha=0.8)\n    ax.bar([i + width/2 for i in x], compressed_scores, width, label='Compressed', alpha=0.8)\n    ax.set_xlabel('Fold')\n    ax.set_ylabel('Pearson Correlation')\n    ax.set_title('Cross-Validation Scores')\n    ax.legend()\n    ax.grid(True, alpha=0.3)\n    \n    # 2. Average performance\n    ax = axes[1]\n    avg_original = np.mean(original_scores)\n    avg_compressed = np.mean(compressed_scores)\n    std_original = np.std(original_scores)\n    std_compressed = np.std(compressed_scores)\n    \n    bars = ax.bar(['Original', 'Compressed'], [avg_original, avg_compressed], \n                   yerr=[std_original, std_compressed], capsize=10)\n    bars[0].set_color('blue')\n    bars[1].set_color('green')\n    \n    ax.set_ylabel('Average Pearson Correlation')\n    ax.set_title('Average Model Performance')\n    ax.grid(True, alpha=0.3)\n    \n    # Add improvement text\n    improvement = ((avg_compressed - avg_original) / avg_original) * 100\n    ax.text(0.5, 0.95, f'Improvement: {improvement:+.1f}%', \n            transform=ax.transAxes, ha='center', fontsize=12,\n            bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.5))\n    \n    # 3. Feature importance comparison\n    ax = axes[2]\n    \n    # Get feature importances from final models\n    model_orig_final = LGBMRegressor(**lgb_params)\n    model_orig_final.fit(X_original, y)\n    importance_orig = model_orig_final.feature_importances_\n    \n    model_comp_final = LGBMRegressor(**lgb_params)\n    model_comp_final.fit(X_compressed, y)\n    importance_comp = model_comp_final.feature_importances_\n    \n    # Plot top 10 features\n    top_indices = np.argsort(importance_orig)[-10:]\n    top_features = [feature_names[i] for i in top_indices]\n    \n    x = np.arange(len(top_features))\n    width = 0.35\n    \n    ax.barh([i - width/2 for i in x], importance_orig[top_indices], width, \n            label='Original', alpha=0.8)\n    ax.barh([i + width/2 for i in x], importance_comp[top_indices], width, \n            label='Compressed', alpha=0.8)\n    \n    ax.set_yticks(x)\n    ax.set_yticklabels(top_features)\n    ax.set_xlabel('Feature Importance')\n    ax.set_title('Top 10 Feature Importances')\n    ax.legend()\n    ax.grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    plt.savefig('compression_impact.png', dpi=150, bbox_inches='tight')\n    plt.show()\n    \n    # Print summary\n    print(f\"\\n📊 COMPRESSION IMPACT SUMMARY:\")\n    print(f\"   Original CV Score: {avg_original:.4f} ± {std_original:.4f}\")\n    print(f\"   Compressed CV Score: {avg_compressed:.4f} ± {std_compressed:.4f}\")\n    print(f\"   Performance Change: {improvement:+.1f}%\")\n    print(f\"   Stability Change: {((std_compressed - std_original) / std_original * 100):+.1f}%\")\n    \n    return {\n        'original_scores': original_scores,\n        'compressed_scores': compressed_scores,\n        'improvement': improvement\n    }\n\n# Main execution\nif __name__ == \"__main__\":\n    # Initialize compressor\n    compressor = EducationalIntelligentCompressor()\n    \n    # Fit compressor\n    compressor.fit(X_train, y_train)\n    \n    # Transform data\n    X_train_compressed = compressor.transform(X_train)\n    \n    # Evaluate impact\n    results = evaluate_compression_impact(\n        X_train, \n        X_train_compressed, \n        y_train,\n        available_features\n    )\n    \n    # Final summary\n    print(\"\\n\" + \"=\"*80)\n    print(\"🎓 EDUCATIONAL SUMMARY: KEY TAKEAWAYS\")\n    print(\"=\"*80)\n    print(\"\\n1. **Intelligent Compression adapts to each feature's characteristics**\")\n    print(\"   - Noisy features → More compression\")\n    print(\"   - Informative features → Less compression\")\n    print(\"   - Outlier-heavy features → Robust methods\")\n    print(\"   - Gradient-sensitive features → Gentle compression\")\n    \n    print(\"\\n2. **Different compression methods for different data patterns**\")\n    print(\"   - Robust Tanh: Heavy-tailed distributions\")\n    print(\"   - Soft Clip: Sensitive feature-target relationships\")\n    print(\"   - Sigmoid: Normal-like distributions\")\n    print(\"   - Log: Skewed distributions\")\n    \n    print(\"\\n3. **Benefits of intelligent compression**\")\n    print(f\"   - Performance improvement: {results['improvement']:+.1f}%\")\n    print(\"   - Noise reduction while preserving signal\")\n    print(\"   - Better generalization through outlier handling\")\n    print(\"   - Adaptive approach beats one-size-fits-all\")\n    \n    print(\"\\n4. **The compression pipeline:**\")\n    print(\"   Analysis → Method Selection → Transformation → Validation\")\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"✅ Educational demonstration complete!\")\n    print(\"Check the generated visualizations for detailed insights.\")\n    print(\"=\"*80)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Educational Version: Prophet-Supervised Intelligent Adaptive Compressor\n# Prophet helps determine optimal compression by analyzing time-series behavior and predicting impact\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import pearsonr, kurtosis, skew\nfrom sklearn.metrics import mutual_info_score\nfrom sklearn.ensemble import RandomForestRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom prophet import Prophet\nfrom prophet.diagnostics import cross_validation, performance_metrics\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set style for beautiful visualizations\nplt.style.use('seaborn-v0_8-darkgrid')\nsns.set_palette(\"husl\")\n\nprint(\"=\"*80)\nprint(\"PROPHET-SUPERVISED INTELLIGENT ADAPTIVE COMPRESSOR\")\nprint(\"=\"*80)\nprint(\"\\nThis enhanced version uses Prophet to:\")\nprint(\"1. Analyze temporal patterns in features\")\nprint(\"2. Predict compression impact on future performance\")\nprint(\"3. Optimize compression strength based on forecasting accuracy\")\nprint(\"4. Validate compression strategies through backtesting\")\nprint(\"=\"*80)\n\n# Configuration\nclass Config:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    n_samples = 20000  # More samples for Prophet\n    n_features = 30\n    seed = 42\n    prophet_horizon = '7 days'  # Forecast horizon\n    prophet_initial = '14 days'  # Initial training period\n    \n# Load data\nprint(\"\\n📊 LOADING DATA FOR PROPHET ANALYSIS...\")\ntrain_df = pd.read_parquet(Config.train_path)\nprint(f\"Full dataset shape: {train_df.shape}\")\n\n# Add timestamp if not present (Prophet needs datetime)\nif 'timestamp' not in train_df.columns:\n    # Create synthetic timestamps for demonstration\n    train_df['timestamp'] = pd.date_range(\n        start='2023-03-01', \n        periods=len(train_df), \n        freq='min'\n    )\n\n# Take recent samples with timestamp\ntrain_df = train_df.tail(Config.n_samples).reset_index(drop=True)\nprint(f\"Using most recent {Config.n_samples} samples\")\nprint(f\"Date range: {train_df['timestamp'].min()} to {train_df['timestamp'].max()}\")\n\n# Select features\nselected_features = [\n    'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume',\n    'X287', 'X446', 'X66', 'X123', 'X385', 'X25', 'X3', \n    'X415', 'X345', 'X37', 'X174', 'X298', 'X178', 'X168',\n    'X598', 'X603', 'X674', 'X756', 'X589', 'X462', 'X779',\n    'X532', 'X520', 'X329', 'X383'\n]\n\navailable_features = [f for f in selected_features if f in train_df.columns]\nX_train = train_df[available_features]\ny_train = train_df['label']\ntimestamps = train_df['timestamp']\n\nprint(f\"\\n✅ Selected {len(available_features)} features for analysis\")\n\nclass ProphetSupervisedCompressor:\n    \"\"\"Prophet-enhanced compressor that uses time-series analysis to optimize compression\"\"\"\n    \n    def __init__(self):\n        self.compression_profiles = {}\n        self.prophet_analyses = {}\n        self.compression_impact_predictions = {}\n        self.optimal_compressions = {}\n        \n    def analyze_temporal_behavior(self, feature_data, timestamps, feature_name):\n        \"\"\"Use Prophet to analyze temporal patterns in the feature\"\"\"\n        print(f\"\\n🔮 PROPHET ANALYSIS: {feature_name}\")\n        print(\"=\"*50)\n        \n        # Prepare data for Prophet\n        prophet_df = pd.DataFrame({\n            'ds': timestamps,\n            'y': feature_data\n        })\n        \n        # Remove NaN values\n        prophet_df = prophet_df.dropna()\n        \n        if len(prophet_df) < 100:\n            print(f\"❌ Insufficient data for Prophet analysis\")\n            return None\n            \n        try:\n            # Fit Prophet model\n            model = Prophet(\n                changepoint_prior_scale=0.05,\n                seasonality_prior_scale=10,\n                daily_seasonality=True,\n                weekly_seasonality=False,\n                yearly_seasonality=False,\n                interval_width=0.95\n            )\n            \n            # Add custom seasonalities if crypto data (24h, 1h patterns)\n            model.add_seasonality(name='hourly', period=1/24, fourier_order=5)\n            \n            with warnings.catch_warnings():\n                warnings.filterwarnings('ignore')\n                model.fit(prophet_df)\n            \n            # Make future dataframe\n            future = model.make_future_dataframe(periods=int(24*7), freq='min')\n            forecast = model.predict(future)\n            \n            # Analyze components\n            fig = model.plot_components(forecast)\n            plt.suptitle(f'Temporal Components of {feature_name}', fontsize=14)\n            plt.tight_layout()\n            plt.savefig(f'prophet_components_{feature_name}.png', dpi=150, bbox_inches='tight')\n            plt.show()\n            \n            # Calculate temporal metrics\n            # 1. Trend strength\n            trend_strength = np.std(forecast['trend']) / (np.std(prophet_df['y']) + 1e-10)\n            \n            # 2. Seasonality strength\n            seasonal_components = ['daily', 'hourly']\n            seasonality_strength = 0\n            for component in seasonal_components:\n                if component in forecast.columns:\n                    seasonality_strength += np.std(forecast[component]) / (np.std(prophet_df['y']) + 1e-10)\n            \n            # 3. Changepoint detection\n            changepoints = model.changepoints\n            n_changepoints = len(changepoints)\n            \n            # 4. Forecast uncertainty\n            uncertainty = np.mean(forecast['yhat_upper'] - forecast['yhat_lower'])\n            \n            print(f\"\\n📈 Temporal Characteristics:\")\n            print(f\"   Trend strength: {trend_strength:.3f}\")\n            print(f\"   Seasonality strength: {seasonality_strength:.3f}\")\n            print(f\"   Number of changepoints: {n_changepoints}\")\n            print(f\"   Forecast uncertainty: {uncertainty:.3f}\")\n            \n            return {\n                'model': model,\n                'forecast': forecast,\n                'trend_strength': trend_strength,\n                'seasonality_strength': seasonality_strength,\n                'n_changepoints': n_changepoints,\n                'uncertainty': uncertainty,\n                'prophet_df': prophet_df\n            }\n            \n        except Exception as e:\n            print(f\"⚠️ Prophet analysis failed: {str(e)}\")\n            return None\n    \n    def test_compression_strategies(self, feature_data, target_data, timestamps, \n                                  feature_name, prophet_analysis):\n        \"\"\"Test different compression strategies and predict their impact using Prophet\"\"\"\n        \n        print(f\"\\n🧪 TESTING COMPRESSION STRATEGIES FOR {feature_name}\")\n        print(\"=\"*60)\n        \n        # Define compression strategies to test\n        compression_strategies = [\n            {'name': 'none', 'strength': 0.0},\n            {'name': 'light', 'strength': 0.2},\n            {'name': 'moderate', 'strength': 0.4},\n            {'name': 'strong', 'strength': 0.6},\n            {'name': 'aggressive', 'strength': 0.8}\n        ]\n        \n        strategy_results = {}\n        \n        # Create figure for strategy comparison\n        fig, axes = plt.subplots(3, 2, figsize=(18, 14))\n        fig.suptitle(f'Compression Strategy Analysis for {feature_name}', fontsize=16)\n        \n        for idx, strategy in enumerate(compression_strategies):\n            print(f\"\\n   Testing {strategy['name']} compression (strength={strategy['strength']})...\")\n            \n            # Apply compression\n            compressed_data = self._apply_test_compression(\n                feature_data, strategy['strength']\n            )\n            \n            # Analyze compressed data with Prophet\n            prophet_df_compressed = pd.DataFrame({\n                'ds': timestamps,\n                'y': compressed_data\n            })\n            prophet_df_compressed = prophet_df_compressed.dropna()\n            \n            try:\n                # Fit Prophet on compressed data\n                model_compressed = Prophet(\n                    changepoint_prior_scale=0.05,\n                    daily_seasonality=True,\n                    interval_width=0.95\n                )\n                model_compressed.add_seasonality(name='hourly', period=1/24, fourier_order=5)\n                \n                with warnings.catch_warnings():\n                    warnings.filterwarnings('ignore')\n                    model_compressed.fit(prophet_df_compressed)\n                \n                # Cross-validation for performance metrics\n                df_cv = cross_validation(\n                    model_compressed, \n                    initial='7 days',\n                    period='1 days',\n                    horizon='1 days',\n                    parallel=\"processes\"\n                )\n                \n                df_p = performance_metrics(df_cv)\n                \n                # Calculate correlation with target after compression\n                mask = ~(np.isnan(compressed_data) | np.isnan(target_data))\n                if np.sum(mask) > 100:\n                    target_correlation = pearsonr(compressed_data[mask], target_data[mask])[0]\n                else:\n                    target_correlation = 0\n                \n                # Store results\n                strategy_results[strategy['name']] = {\n                    'mape': df_p['mape'].mean(),\n                    'rmse': df_p['rmse'].mean(),\n                    'mae': df_p['mae'].mean(),\n                    'forecast_accuracy': 1 - df_p['mape'].mean(),\n                    'target_correlation': target_correlation,\n                    'compression_strength': strategy['strength']\n                }\n                \n                print(f\"      MAPE: {df_p['mape'].mean():.3f}\")\n                print(f\"      Target correlation: {target_correlation:.3f}\")\n                \n            except Exception as e:\n                print(f\"      ⚠️ Strategy evaluation failed: {str(e)}\")\n                strategy_results[strategy['name']] = {\n                    'mape': 1.0,\n                    'rmse': np.inf,\n                    'mae': np.inf,\n                    'forecast_accuracy': 0,\n                    'target_correlation': 0,\n                    'compression_strength': strategy['strength']\n                }\n        \n        # Visualize strategy comparison\n        # 1. Forecast accuracy by compression\n        ax = axes[0, 0]\n        strategies = list(strategy_results.keys())\n        accuracies = [strategy_results[s]['forecast_accuracy'] for s in strategies]\n        correlations = [strategy_results[s]['target_correlation'] for s in strategies]\n        \n        ax.plot(strategies, accuracies, 'o-', label='Forecast Accuracy', linewidth=2, markersize=8)\n        ax.plot(strategies, correlations, 's-', label='Target Correlation', linewidth=2, markersize=8)\n        ax.set_xlabel('Compression Strategy')\n        ax.set_ylabel('Score')\n        ax.set_title('Performance Metrics by Compression Strategy')\n        ax.legend()\n        ax.grid(True, alpha=0.3)\n        \n        # 2. Error metrics comparison\n        ax = axes[0, 1]\n        mapes = [strategy_results[s]['mape'] for s in strategies]\n        maes = [strategy_results[s]['mae'] for s in strategies]\n        \n        width = 0.35\n        x = np.arange(len(strategies))\n        ax.bar(x - width/2, mapes, width, label='MAPE', alpha=0.8)\n        ax.bar(x + width/2, maes, width, label='MAE', alpha=0.8)\n        ax.set_xlabel('Compression Strategy')\n        ax.set_ylabel('Error')\n        ax.set_title('Forecast Error by Strategy')\n        ax.set_xticks(x)\n        ax.set_xticklabels(strategies)\n        ax.legend()\n        ax.grid(True, alpha=0.3)\n        \n        # 3. Optimal compression determination\n        ax = axes[1, 0]\n        \n        # Create composite score\n        composite_scores = []\n        for s in strategies:\n            # Higher forecast accuracy and correlation are better\n            # Lower MAPE is better\n            score = (strategy_results[s]['forecast_accuracy'] * 0.4 + \n                    strategy_results[s]['target_correlation'] * 0.4 +\n                    (1 - strategy_results[s]['mape']) * 0.2)\n            composite_scores.append(score)\n        \n        ax.bar(strategies, composite_scores, color='green', alpha=0.7)\n        ax.set_xlabel('Compression Strategy')\n        ax.set_ylabel('Composite Score')\n        ax.set_title('Overall Strategy Performance')\n        ax.grid(True, alpha=0.3)\n        \n        # Highlight optimal\n        optimal_idx = np.argmax(composite_scores)\n        ax.bar(optimal_idx, composite_scores[optimal_idx], color='darkgreen')\n        ax.annotate('OPTIMAL', xy=(optimal_idx, composite_scores[optimal_idx]), \n                   xytext=(optimal_idx, composite_scores[optimal_idx] + 0.05),\n                   ha='center', fontsize=12, fontweight='bold')\n        \n        # 4. Compression impact on signal\n        ax = axes[1, 1]\n        compressions = [strategy_results[s]['compression_strength'] for s in strategies]\n        \n        ax.plot(compressions, accuracies, 'o-', label='Forecast Accuracy', linewidth=2)\n        ax.plot(compressions, correlations, 's-', label='Target Correlation', linewidth=2)\n        ax.set_xlabel('Compression Strength')\n        ax.set_ylabel('Score')\n        ax.set_title('Performance vs Compression Strength')\n        ax.legend()\n        ax.grid(True, alpha=0.3)\n        \n        # 5. Time series comparison (original vs optimal)\n        ax = axes[2, 0]\n        optimal_strategy = compression_strategies[optimal_idx]\n        optimal_compressed = self._apply_test_compression(\n            feature_data[-1000:], optimal_strategy['strength']\n        )\n        \n        time_idx = range(len(optimal_compressed))\n        ax.plot(time_idx, feature_data[-1000:], alpha=0.5, label='Original')\n        ax.plot(time_idx, optimal_compressed, alpha=0.8, label=f\"Optimal ({optimal_strategy['name']})\")\n        ax.set_xlabel('Time')\n        ax.set_ylabel('Value')\n        ax.set_title('Original vs Optimally Compressed Signal')\n        ax.legend()\n        ax.grid(True, alpha=0.3)\n        \n        # 6. Prophet recommendation\n        ax = axes[2, 1]\n        ax.text(0.5, 0.5, f\"PROPHET RECOMMENDATION\\n\\n\"\n                f\"Feature: {feature_name}\\n\"\n                f\"Optimal Strategy: {optimal_strategy['name']}\\n\"\n                f\"Compression Strength: {optimal_strategy['strength']:.2f}\\n\"\n                f\"Expected Forecast Accuracy: {accuracies[optimal_idx]:.3f}\\n\"\n                f\"Expected Target Correlation: {correlations[optimal_idx]:.3f}\\n\\n\"\n                f\"Reasoning: This compression level maintains\\n\"\n                f\"the best balance between noise reduction\\n\"\n                f\"and signal preservation for forecasting.\",\n                transform=ax.transAxes, ha='center', va='center',\n                fontsize=12, bbox=dict(boxstyle='round', facecolor='lightblue', alpha=0.8))\n        ax.axis('off')\n        \n        plt.tight_layout()\n        plt.savefig(f'prophet_strategy_analysis_{feature_name}.png', dpi=150, bbox_inches='tight')\n        plt.show()\n        \n        # Return optimal compression\n        optimal_compression = optimal_strategy['strength']\n        \n        print(f\"\\n✅ PROPHET RECOMMENDATION: {optimal_strategy['name']} compression\")\n        print(f\"   Optimal strength: {optimal_compression:.2f}\")\n        print(f\"   Expected performance: {composite_scores[optimal_idx]:.3f}\")\n        \n        return optimal_compression, strategy_results\n    \n    def _apply_test_compression(self, data, strength):\n        \"\"\"Apply simple compression for testing\"\"\"\n        if strength == 0:\n            return data\n            \n        # Simple tanh compression\n        median = np.nanmedian(data)\n        mad = np.nanmedian(np.abs(data - median))\n        \n        if mad == 0:\n            return data\n            \n        normalized = (data - median) / (mad * 6)\n        compressed = np.tanh(normalized * (1 - strength))\n        \n        return compressed * (mad * 6) + median\n    \n    def analyze_feature_with_prophet(self, data, target, timestamps, feature_name):\n        \"\"\"Complete Prophet-supervised analysis of a feature\"\"\"\n        \n        # First, do temporal analysis\n        prophet_analysis = self.analyze_temporal_behavior(data, timestamps, feature_name)\n        \n        if prophet_analysis is None:\n            return None\n            \n        # Test compression strategies\n        optimal_compression, strategy_results = self.test_compression_strategies(\n            data, target, timestamps, feature_name, prophet_analysis\n        )\n        \n        # Store results\n        self.prophet_analyses[feature_name] = prophet_analysis\n        self.compression_impact_predictions[feature_name] = strategy_results\n        self.optimal_compressions[feature_name] = optimal_compression\n        \n        # Now do traditional analysis with Prophet-informed compression\n        analysis_result = self._traditional_analysis(data, target, feature_name)\n        \n        # Override compression with Prophet recommendation\n        analysis_result['optimal_compression'] = optimal_compression\n        analysis_result['prophet_informed'] = True\n        analysis_result['temporal_metrics'] = {\n            'trend_strength': prophet_analysis['trend_strength'],\n            'seasonality_strength': prophet_analysis['seasonality_strength'],\n            'changepoints': prophet_analysis['n_changepoints'],\n            'forecast_uncertainty': prophet_analysis['uncertainty']\n        }\n        \n        return analysis_result\n    \n    def _traditional_analysis(self, data, target, feature_name):\n        \"\"\"Traditional feature analysis (simplified for brevity)\"\"\"\n        # Remove NaN values\n        mask = ~(np.isnan(data) | np.isnan(target))\n        clean_data = data[mask]\n        clean_target = target[mask]\n        \n        if len(clean_data) < 10:\n            return None\n            \n        # Basic statistics\n        median = np.median(clean_data)\n        mad = np.median(np.abs(clean_data - median))\n        \n        # Noise level\n        if len(clean_data) > 100:\n            fft = np.fft.fft(clean_data)\n            frequencies = np.abs(fft)\n            noise_level = np.sum(frequencies[len(frequencies)//2:]) / np.sum(frequencies)\n        else:\n            noise_level = 0.5\n            \n        # Outlier score\n        if mad > 0:\n            z_scores = np.abs((clean_data - median) / mad)\n            outlier_score = np.mean(z_scores > 3)\n        else:\n            outlier_score = 0\n            \n        # Distribution score\n        try:\n            kurt = kurtosis(clean_data)\n            skewness = abs(skew(clean_data))\n            distribution_score = 1 / (1 + np.exp(-(kurt / 10 + skewness / 5)))\n        except:\n            distribution_score = 0.5\n            \n        # Gradient sensitivity\n        try:\n            sorted_idx = np.argsort(clean_data)\n            sorted_feature = clean_data[sorted_idx]\n            sorted_target = clean_target[sorted_idx]\n            \n            feature_diffs = np.diff(sorted_feature)\n            target_diffs = np.diff(sorted_target)\n            \n            mask = feature_diffs != 0\n            if np.any(mask):\n                gradients = np.abs(target_diffs[mask] / feature_diffs[mask])\n                gradient_sensitivity = np.percentile(gradients, 90)\n            else:\n                gradient_sensitivity = 0\n        except:\n            gradient_sensitivity = 0\n            \n        # Mutual information\n        try:\n            n_bins = min(20, len(clean_data) // 10)\n            feature_bins = pd.qcut(clean_data, n_bins, labels=False, duplicates='drop')\n            target_bins = pd.qcut(clean_target, n_bins, labels=False, duplicates='drop')\n            \n            mutual_info = mutual_info_score(feature_bins, target_bins)\n            max_entropy = np.log2(n_bins)\n            mutual_info_score_norm = mutual_info / max_entropy if max_entropy > 0 else 0\n        except:\n            mutual_info_score_norm = 0\n            \n        # Method selection based on characteristics\n        if outlier_score > 0.15 or distribution_score > 0.7:\n            method = 'robust_tanh'\n        elif gradient_sensitivity > 5:\n            method = 'soft_clip'\n        elif abs(skewness) > 2:\n            method = 'log'\n        else:\n            method = 'sigmoid'\n            \n        return {\n            'median': median,\n            'mad': mad,\n            'noise_level': noise_level,\n            'outlier_score': outlier_score,\n            'distribution_score': distribution_score,\n            'gradient_sensitivity': gradient_sensitivity,\n            'mutual_info_score': mutual_info_score_norm,\n            'method': method\n        }\n    \n    def fit(self, X, y, timestamps):\n        \"\"\"Fit compressor with Prophet supervision\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"🚀 FITTING PROPHET-SUPERVISED INTELLIGENT COMPRESSOR\")\n        print(\"=\"*80)\n        \n        # Analyze first 5 features in detail\n        for i, col in enumerate(X.columns[:5]):\n            analysis = self.analyze_feature_with_prophet(\n                X[col].values,\n                y.values,\n                timestamps,\n                col\n            )\n            \n            if analysis:\n                self.compression_profiles[col] = analysis\n        \n        # Quick analysis for remaining features\n        print(\"\\n⏭️ Quick Prophet analysis for remaining features...\")\n        for col in X.columns[5:]:\n            # Just get Prophet recommendation without full visualization\n            prophet_analysis = self.analyze_temporal_behavior(\n                X[col].values, timestamps, col\n            )\n            \n            if prophet_analysis:\n                # Simplified strategy testing\n                optimal_compression = self._quick_prophet_optimization(\n                    X[col].values, y.values, timestamps, prophet_analysis\n                )\n                \n                analysis = self._traditional_analysis(\n                    X[col].values, y.values, col\n                )\n                \n                if analysis:\n                    analysis['optimal_compression'] = optimal_compression\n                    analysis['prophet_informed'] = True\n                    self.compression_profiles[col] = analysis\n        \n        # Create summary\n        self._create_prophet_summary()\n        \n        return self\n    \n    def _quick_prophet_optimization(self, data, target, timestamps, prophet_analysis):\n        \"\"\"Quick Prophet-based optimization without full visualization\"\"\"\n        # Use Prophet uncertainty to guide compression\n        uncertainty = prophet_analysis['uncertainty']\n        trend_strength = prophet_analysis['trend_strength']\n        seasonality_strength = prophet_analysis['seasonality_strength']\n        \n        # Higher uncertainty → more compression\n        # Stronger trend/seasonality → less compression (preserve patterns)\n        base_compression = min(0.8, uncertainty / (np.std(data) + 1e-10))\n        pattern_preservation = (trend_strength + seasonality_strength) / 2\n        \n        optimal_compression = base_compression * (1 - pattern_preservation * 0.5)\n        \n        return np.clip(optimal_compression, 0, 0.8)\n    \n    def _create_prophet_summary(self):\n        \"\"\"Create Prophet-informed summary visualization\"\"\"\n        if not self.compression_profiles:\n            return\n            \n        # Prepare summary data\n        summary_data = []\n        for feature, profile in self.compression_profiles.items():\n            row = {\n                'Feature': feature,\n                'Compression': profile['optimal_compression'],\n                'Method': profile['method'],\n                'Prophet_Informed': profile.get('prophet_informed', False)\n            }\n            \n            if 'temporal_metrics' in profile:\n                row.update({\n                    'Trend': profile['temporal_metrics']['trend_strength'],\n                    'Seasonality': profile['temporal_metrics']['seasonality_strength'],\n                    'Changepoints': profile['temporal_metrics']['changepoints']\n                })\n            \n            summary_data.append(row)\n        \n        summary_df = pd.DataFrame(summary_data)\n        \n        # Create visualization\n        fig, axes = plt.subplots(2, 2, figsize=(16, 12))\n        fig.suptitle('Prophet-Supervised Compression Summary', fontsize=16)\n        \n        # 1. Compression by feature\n        ax = axes[0, 0]\n        prophet_features = summary_df[summary_df['Prophet_Informed'] == True]\n        if len(prophet_features) > 0:\n            prophet_features = prophet_features.sort_values('Compression', ascending=False).head(15)\n            bars = ax.bar(prophet_features['Feature'], prophet_features['Compression'])\n            \n            # Color by Prophet recommendation\n            for i, bar in enumerate(bars):\n                if prophet_features.iloc[i]['Compression'] < 0.3:\n                    bar.set_color('green')\n                elif prophet_features.iloc[i]['Compression'] < 0.6:\n                    bar.set_color('orange')\n                else:\n                    bar.set_color('red')\n            \n            ax.set_title('Prophet-Recommended Compression Strengths')\n            ax.set_xlabel('Feature')\n            ax.set_ylabel('Compression Strength')\n            ax.set_xticklabels(prophet_features['Feature'], rotation=45, ha='right')\n        \n        # 2. Temporal patterns vs compression\n        ax = axes[0, 1]\n        if 'Trend' in summary_df.columns and 'Seasonality' in summary_df.columns:\n            temporal_strength = summary_df['Trend'] + summary_df['Seasonality']\n            scatter = ax.scatter(temporal_strength, summary_df['Compression'], \n                               s=100, alpha=0.6, c=summary_df['Compression'], cmap='RdYlGn_r')\n            ax.set_xlabel('Temporal Pattern Strength (Trend + Seasonality)')\n            ax.set_ylabel('Recommended Compression')\n            ax.set_title('Temporal Patterns vs Compression')\n            plt.colorbar(scatter, ax=ax, label='Compression')\n            \n            # Add trend line\n            z = np.polyfit(temporal_strength.dropna(), \n                          summary_df.loc[temporal_strength.notna(), 'Compression'], 1)\n            p = np.poly1d(z)\n            ax.plot(np.sort(temporal_strength.dropna()), \n                   p(np.sort(temporal_strength.dropna())), \n                   \"r--\", alpha=0.8, label='Trend')\n            ax.legend()\n        \n        # 3. Method distribution\n        ax = axes[1, 0]\n        method_counts = summary_df['Method'].value_counts()\n        ax.pie(method_counts.values, labels=method_counts.index, autopct='%1.1f%%')\n        ax.set_title('Distribution of Compression Methods')\n        \n        # 4. Prophet insights\n        ax = axes[1, 1]\n        insights = [\n            \"PROPHET SUPERVISION INSIGHTS:\",\n            \"\",\n            f\"✓ {len(prophet_features)} features analyzed with Prophet\",\n            f\"✓ Average compression: {summary_df['Compression'].mean():.3f}\",\n            \"\",\n            \"Key Findings:\",\n            \"• High temporal patterns → Lower compression\",\n            \"• High uncertainty → Higher compression\", \n            \"• Changepoints indicate need for adaptive compression\",\n            \"\",\n            \"Benefits:\",\n            \"• Time-aware compression optimization\",\n            \"• Preserves important temporal patterns\",\n            \"• Better forecasting performance\"\n        ]\n        \n        ax.text(0.1, 0.9, '\\n'.join(insights), transform=ax.transAxes,\n               fontsize=11, verticalalignment='top',\n               bbox=dict(boxstyle='round', facecolor='lightblue', alpha=0.8))\n        ax.axis('off')\n        \n        plt.tight_layout()\n        plt.savefig('prophet_compression_summary.png', dpi=150, bbox_inches='tight')\n        plt.show()\n        \n        print(\"\\n\" + \"=\"*80)\n        print(\"📊 PROPHET SUPERVISION SUMMARY\")\n        print(\"=\"*80)\n        print(f\"\\nFeatures with temporal analysis: {len(prophet_features)}\")\n        print(f\"Average Prophet-recommended compression: {summary_df['Compression'].mean():.3f}\")\n        \n        if 'Trend' in summary_df.columns:\n            high_temporal = summary_df[\n                (summary_df['Trend'] > 0.5) | (summary_df['Seasonality'] > 0.5)\n            ]\n            print(f\"\\nHigh temporal pattern features: {len(high_temporal)}\")\n            if len(high_temporal) > 0:\n                print(f\"Examples: {', '.join(high_temporal['Feature'].head(5).tolist())}\")\n\ndef prophet_supervised_pipeline(X_train, y_train, timestamps):\n    \"\"\"Main pipeline with Prophet supervision\"\"\"\n    \n    # Initialize Prophet-supervised compressor\n    compressor = ProphetSupervisedCompressor()\n    \n    # Fit with Prophet supervision\n    compressor.fit(X_train, y_train, timestamps)\n    \n    # Apply compression\n    X_compressed = X_train.copy()\n    \n    for col in X_train.columns:\n        if col in compressor.compression_profiles:\n            profile = compressor.compression_profiles[col]\n            \n            # Apply compression based on Prophet recommendation\n            median = profile['median']\n            mad = profile['mad']\n            strength = profile['optimal_compression']\n            \n            if mad > 0 and strength > 0:\n                normalized = (X_train[col] - median) / (mad * 6)\n                \n                if profile['method'] == 'robust_tanh':\n                    compressed = np.tanh(normalized * (1 - strength))\n                elif profile['method'] == 'soft_clip':\n                    threshold = 2 * (1 - strength + 0.1)\n                    compressed = np.where(\n                        np.abs(normalized) < threshold,\n                        normalized,\n                        np.sign(normalized) * (threshold + np.log1p(np.abs(normalized) - threshold) * 0.5)\n                    )\n                else:\n                    compressed = np.tanh(normalized * (1 - strength))\n                \n                X_compressed[col] = compressed * (mad * 6) + median\n    \n    return X_compressed, compressor\n\ndef evaluate_prophet_impact(X_original, X_compressed, y, timestamps):\n    \"\"\"Evaluate Prophet-supervised compression with time-aware validation\"\"\"\n    print(\"\\n\" + \"=\"*80)\n    print(\"📈 EVALUATING PROPHET-SUPERVISED COMPRESSION IMPACT\")\n    print(\"=\"*80)\n    \n    # Time series split for proper temporal validation\n    tscv = TimeSeriesSplit(n_splits=5)\n    \n    lgb_params = {\n        'n_estimators': 100,\n        'learning_rate': 0.1,\n        'num_leaves': 31,\n        'random_state': 42,\n        'verbose': -1\n    }\n    \n    original_scores = []\n    compressed_scores = []\n    \n    print(\"\\n🔄 Running time-series cross-validation...\")\n    \n    for fold, (train_idx, val_idx) in enumerate(tscv.split(X_original)):\n        # Ensure temporal order\n        train_dates = timestamps.iloc[train_idx]\n        val_dates = timestamps.iloc[val_idx]\n        \n        print(f\"\\n   Fold {fold+1}:\")\n        print(f\"   Train: {train_dates.min()} to {train_dates.max()}\")\n        print(f\"   Val: {val_dates.min()} to {val_dates.max()}\")\n        \n        # Original data\n        model_orig = LGBMRegressor(**lgb_params)\n        model_orig.fit(X_original.iloc[train_idx], y.iloc[train_idx])\n        pred_orig = model_orig.predict(X_original.iloc[val_idx])\n        score_orig = pearsonr(y.iloc[val_idx], pred_orig)[0]\n        original_scores.append(score_orig)\n        \n        # Compressed data\n        model_comp = LGBMRegressor(**lgb_params)\n        model_comp.fit(X_compressed.iloc[train_idx], y.iloc[train_idx])\n        pred_comp = model_comp.predict(X_compressed.iloc[val_idx])\n        score_comp = pearsonr(y.iloc[val_idx], pred_comp)[0]\n        compressed_scores.append(score_comp)\n        \n        print(f\"   Original: {score_orig:.4f}, Compressed: {score_comp:.4f}\")\n    \n    # Visualize temporal validation results\n    fig, axes = plt.subplots(2, 2, figsize=(16, 10))\n    fig.suptitle('Prophet-Supervised Compression: Temporal Validation', fontsize=16)\n    \n    # 1. Score progression over folds\n    ax = axes[0, 0]\n    folds = range(1, len(original_scores) + 1)\n    ax.plot(folds, original_scores, 'o-', label='Original', linewidth=2, markersize=8)\n    ax.plot(folds, compressed_scores, 's-', label='Prophet-Compressed', linewidth=2, markersize=8)\n    ax.set_xlabel('Time-Based Fold')\n    ax.set_ylabel('Pearson Correlation')\n    ax.set_title('Performance Over Time')\n    ax.legend()\n    ax.grid(True, alpha=0.3)\n    \n    # 2. Improvement by fold\n    ax = axes[0, 1]\n    improvements = [(c - o) / o * 100 for o, c in zip(original_scores, compressed_scores)]\n    bars = ax.bar(folds, improvements)\n    \n    # Color positive/negative differently\n    for bar, imp in zip(bars, improvements):\n        bar.set_color('green' if imp > 0 else 'red')\n    \n    ax.axhline(y=0, color='black', linestyle='-', linewidth=0.5)\n    ax.set_xlabel('Time-Based Fold')\n    ax.set_ylabel('Improvement (%)')\n    ax.set_title('Compression Impact Over Time')\n    ax.grid(True, alpha=0.3)\n    \n    # 3. Average performance comparison\n    ax = axes[1, 0]\n    avg_original = np.mean(original_scores)\n    avg_compressed = np.mean(compressed_scores)\n    std_original = np.std(original_scores)\n    std_compressed = np.std(compressed_scores)\n    \n    methods = ['Original', 'Prophet-Compressed']\n    avgs = [avg_original, avg_compressed]\n    stds = [std_original, std_compressed]\n    \n    bars = ax.bar(methods, avgs, yerr=stds, capsize=10)\n    bars[0].set_color('blue')\n    bars[1].set_color('green')\n    \n    ax.set_ylabel('Average Pearson Correlation')\n    ax.set_title('Overall Performance Comparison')\n    \n    improvement = ((avg_compressed - avg_original) / avg_original) * 100\n    ax.text(0.5, 0.95, f'Improvement: {improvement:+.1f}%', \n            transform=ax.transAxes, ha='center', fontsize=14,\n            bbox=dict(boxstyle='round', facecolor='yellow', alpha=0.5))\n    \n    # 4. Stability analysis\n    ax = axes[1, 1]\n    \n    # Calculate rolling correlation stability\n    window = 2\n    if len(original_scores) >= window:\n        rolling_std_orig = pd.Series(original_scores).rolling(window).std().dropna()\n        rolling_std_comp = pd.Series(compressed_scores).rolling(window).std().dropna()\n        \n        ax.plot(range(len(rolling_std_orig)), rolling_std_orig, 'o-', \n                label='Original', linewidth=2)\n        ax.plot(range(len(rolling_std_comp)), rolling_std_comp, 's-', \n                label='Prophet-Compressed', linewidth=2)\n        ax.set_xlabel('Rolling Window')\n        ax.set_ylabel('Standard Deviation')\n        ax.set_title('Prediction Stability (Lower is Better)')\n        ax.legend()\n        ax.grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    plt.savefig('prophet_temporal_validation.png', dpi=150, bbox_inches='tight')\n    plt.show()\n    \n    # Print summary\n    print(f\"\\n📊 PROPHET-SUPERVISED COMPRESSION SUMMARY:\")\n    print(f\"   Original CV Score: {avg_original:.4f} ± {std_original:.4f}\")\n    print(f\"   Compressed CV Score: {avg_compressed:.4f} ± {std_compressed:.4f}\")\n    print(f\"   Performance Change: {improvement:+.1f}%\")\n    print(f\"   Stability Improvement: {((std_original - std_compressed) / std_original * 100):+.1f}%\")\n    \n    return {\n        'original_scores': original_scores,\n        'compressed_scores': compressed_scores,\n        'improvement': improvement\n    }\n\n# Main execution\nif __name__ == \"__main__\":\n    print(\"\\n🎯 STARTING PROPHET-SUPERVISED COMPRESSION ANALYSIS\")\n    print(\"=\"*80)\n    \n    # Apply Prophet-supervised compression\n    X_compressed, compressor = prophet_supervised_pipeline(\n        X_train, y_train, timestamps\n    )\n    \n    # Evaluate impact with temporal validation\n    results = evaluate_prophet_impact(\n        X_train, X_compressed, y_train, timestamps\n    )\n    \n    # Final insights\n    print(\"\\n\" + \"=\"*80)\n    print(\"🎓 PROPHET SUPERVISION: KEY INSIGHTS\")\n    print(\"=\"*80)\n    \n    print(\"\\n1. **Prophet adds temporal awareness to compression**\")\n    print(\"   - Analyzes trends, seasonality, and changepoints\")\n    print(\"   - Preserves important temporal patterns\")\n    print(\"   - Adapts compression to forecast uncertainty\")\n    \n    print(\"\\n2. **Time-based optimization beats static approaches**\")\n    print(\"   - Tests multiple compression strategies\")\n    print(\"   - Validates with proper time-series splits\")\n    print(\"   - Selects optimal compression per feature\")\n    \n    print(\"\\n3. **Benefits of Prophet supervision:**\")\n    print(f\"   - Performance improvement: {results['improvement']:+.1f}%\")\n    print(\"   - Better temporal pattern preservation\")\n    print(\"   - More stable predictions over time\")\n    print(\"   - Forecast-aware compression decisions\")\n    \n    print(\"\\n4. **The Prophet-enhanced pipeline:**\")\n    print(\"   Temporal Analysis → Strategy Testing → \")\n    print(\"   Impact Prediction → Optimal Selection → Validation\")\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"✅ Prophet-supervised compression analysis complete!\")\n    print(\"Check generated visualizations for detailed insights.\")\n    print(\"=\"*80)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}