{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-17T01:54:38.379412Z","iopub.execute_input":"2025-06-17T01:54:38.379621Z","iopub.status.idle":"2025-06-17T01:54:38.788106Z","shell.execute_reply.started":"2025-06-17T01:54:38.379595Z","shell.execute_reply":"2025-06-17T01:54:38.787399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install numpy pandas matplotlib seaborn scikit-learn umap-learn tensorflow minisom\n#!/usr/bin/env python3\n\"\"\"\nComprehensive DRW Crypto Dimensionality Reduction Analysis\n=========================================================\n\nComplete evaluation of 20+ dimensionality reduction techniques on the actual DRW crypto dataset.\nIncludes robust preprocessing, outlier handling, and evaluation metrics tailored for financial prediction.\n\"\"\"\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Core ML libraries\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, PowerTransformer, QuantileTransformer\nfrom sklearn.decomposition import PCA, KernelPCA, FactorAnalysis, FastICA, NMF, DictionaryLearning\nfrom sklearn.manifold import TSNE, Isomap, LocallyLinearEmbedding, MDS, SpectralEmbedding\nfrom sklearn.random_projection import GaussianRandomProjection\nfrom sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression, VarianceThreshold\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.linear_model import ElasticNet, Ridge\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.neighbors import NearestNeighbors\nfrom scipy import stats\nfrom scipy.stats import pearsonr\n\n# Advanced libraries\nimport umap\ntry:\n    import tensorflow as tf\n    from tensorflow import keras\n    from tensorflow.keras import layers, Model, regularizers\n    tf.random.set_seed(42)\n    TENSORFLOW_AVAILABLE = True\nexcept ImportError:\n    TENSORFLOW_AVAILABLE = False\n    print(\"TensorFlow not available - deep learning methods will be skipped\")\n\ntry:\n    from minisom import MiniSom\n    MINISOM_AVAILABLE = True\nexcept ImportError:\n    MINISOM_AVAILABLE = False\n    print(\"MiniSom not available - SOM will be skipped\")\n\n# Set random seed\nnp.random.seed(42)\n\nclass CryptoDataPreprocessor:\n    \"\"\"Specialized preprocessing for crypto financial data\"\"\"\n    \n    def __init__(self):\n        self.scaler = None\n        self.variance_selector = None\n        self.outlier_method = None\n        self.feature_selector = None\n        \n    def handle_infinite_values(self, X, method='clip'):\n        \"\"\"Handle infinite and NaN values in financial data\"\"\"\n        \n        if method == 'clip':\n            # Replace inf with very large but finite values\n            X = np.where(np.isposinf(X), np.finfo(np.float64).max/1e6, X)\n            X = np.where(np.isneginf(X), np.finfo(np.float64).min/1e6, X)\n            X = np.nan_to_num(X, nan=0.0)\n            \n        elif method == 'percentile':\n            # Replace inf with 99.9th percentile values\n            for col in range(X.shape[1]):\n                finite_mask = np.isfinite(X[:, col])\n                if np.any(finite_mask):\n                    p99 = np.percentile(X[finite_mask, col], 99.9)\n                    p01 = np.percentile(X[finite_mask, col], 0.1)\n                    X[np.isposinf(X[:, col]), col] = p99\n                    X[np.isneginf(X[:, col]), col] = p01\n                    X[np.isnan(X[:, col]), col] = np.median(X[finite_mask, col])\n                    \n        elif method == 'remove':\n            # Remove rows with infinite values\n            finite_mask = np.all(np.isfinite(X), axis=1)\n            X = X[finite_mask]\n            return X, finite_mask\n            \n        return X, None\n    \n    def handle_outliers(self, X, method='iqr', factor=3.0):\n        \"\"\"Handle outliers in financial data\"\"\"\n        \n        if method == 'iqr':\n            Q1 = np.percentile(X, 25, axis=0)\n            Q3 = np.percentile(X, 75, axis=0)\n            IQR = Q3 - Q1\n            lower_bound = Q1 - factor * IQR\n            upper_bound = Q3 + factor * IQR\n            X = np.clip(X, lower_bound, upper_bound)\n            \n        elif method == 'zscore':\n            z_scores = np.abs(stats.zscore(X, axis=0, nan_policy='omit'))\n            X = np.where(z_scores > factor, \n                        np.sign(X) * factor * np.std(X, axis=0) + np.mean(X, axis=0), X)\n            \n        elif method == 'percentile':\n            lower_p = (100 - factor * 10) / 2\n            upper_p = 100 - lower_p\n            lower_bound = np.percentile(X, lower_p, axis=0)\n            upper_bound = np.percentile(X, upper_p, axis=0)\n            X = np.clip(X, lower_bound, upper_bound)\n            \n        elif method == 'none':\n            pass  # No outlier handling\n            \n        return X\n    \n    def select_preprocessing_strategy(self, X, y=None):\n        \"\"\"Grid search for best preprocessing strategy\"\"\"\n        \n        strategies = [\n            {'inf_method': 'clip', 'outlier_method': 'iqr', 'scaler': 'robust'},\n            {'inf_method': 'clip', 'outlier_method': 'percentile', 'scaler': 'robust'},\n            {'inf_method': 'percentile', 'outlier_method': 'iqr', 'scaler': 'standard'},\n            {'inf_method': 'percentile', 'outlier_method': 'zscore', 'scaler': 'robust'},\n            {'inf_method': 'clip', 'outlier_method': 'none', 'scaler': 'quantile'},\n        ]\n        \n        best_score = -np.inf\n        best_strategy = strategies[0]\n        \n        if y is not None:\n            print(\"Testing preprocessing strategies...\")\n            \n            for i, strategy in enumerate(strategies):\n                try:\n                    X_processed = X.copy()\n                    \n                    # Handle infinite values\n                    X_processed, mask = self.handle_infinite_values(X_processed, strategy['inf_method'])\n                    if mask is not None:\n                        y_temp = y[mask]\n                    else:\n                        y_temp = y\n                    \n                    # Handle outliers\n                    X_processed = self.handle_outliers(X_processed, strategy['outlier_method'])\n                    \n                    # Scale data\n                    if strategy['scaler'] == 'robust':\n                        scaler = RobustScaler()\n                    elif strategy['scaler'] == 'standard':\n                        scaler = StandardScaler()\n                    elif strategy['scaler'] == 'quantile':\n                        scaler = QuantileTransformer()\n                    else:\n                        scaler = StandardScaler()\n                    \n                    X_scaled = scaler.fit_transform(X_processed)\n                    \n                    # Quick validation with simple model\n                    if len(X_scaled) > 100:  # Ensure we have enough data\n                        rf = RandomForestRegressor(n_estimators=50, random_state=42)\n                        scores = cross_val_score(rf, X_scaled, y_temp, cv=3, scoring='r2')\n                        score = np.mean(scores)\n                        \n                        if score > best_score:\n                            best_score = score\n                            best_strategy = strategy\n                            \n                        print(f\"Strategy {i+1}: R² = {score:.4f}\")\n                        \n                except Exception as e:\n                    print(f\"Strategy {i+1}: Failed - {str(e)}\")\n                    continue\n        \n        print(f\"Best strategy: {best_strategy} (R² = {best_score:.4f})\")\n        return best_strategy\n    \n    def fit_transform(self, X, y=None, strategy=None):\n        \"\"\"Fit preprocessor and transform data\"\"\"\n        \n        if strategy is None:\n            strategy = self.select_preprocessing_strategy(X, y)\n        \n        # Handle infinite values\n        X_processed, mask = self.handle_infinite_values(X, strategy['inf_method'])\n        \n        # Handle outliers\n        X_processed = self.handle_outliers(X_processed, strategy['outlier_method'])\n        \n        # Remove low variance features\n        self.variance_selector = VarianceThreshold(threshold=0.0)\n        X_processed = self.variance_selector.fit_transform(X_processed)\n        \n        # Scale data\n        if strategy['scaler'] == 'robust':\n            self.scaler = RobustScaler()\n        elif strategy['scaler'] == 'standard':\n            self.scaler = StandardScaler()\n        elif strategy['scaler'] == 'quantile':\n            self.scaler = QuantileTransformer()\n        else:\n            self.scaler = StandardScaler()\n        \n        X_scaled = self.scaler.fit_transform(X_processed)\n        \n        return X_scaled, mask\n    \n    def transform(self, X):\n        \"\"\"Transform new data using fitted preprocessor\"\"\"\n        X_processed, _ = self.handle_infinite_values(X, self.inf_method)\n        X_processed = self.handle_outliers(X_processed, self.outlier_method)\n        X_processed = self.variance_selector.transform(X_processed)\n        X_scaled = self.scaler.transform(X_processed)\n        return X_scaled\n\n\nclass DimensionalityReductionEvaluator:\n    \"\"\"Comprehensive evaluation of dimensionality reduction methods\"\"\"\n    \n    def __init__(self, target_dims=50):\n        self.target_dims = target_dims\n        self.results = {}\n        self.evaluation_metrics = {}\n        self.fitted_reducers = {}\n        \n    def evaluate_predictive_power(self, X_original, X_reduced, y, method_name):\n        \"\"\"Evaluate how well reduced features predict the target\"\"\"\n        \n        if len(X_reduced) != len(y):\n            print(f\"Warning: {method_name} changed sample size\")\n            return {}\n        \n        # Split data\n        X_orig_train, X_orig_test, X_red_train, X_red_test, y_train, y_test = \\\n            train_test_split(X_original, X_reduced, y, test_size=0.3, random_state=42)\n        \n        metrics = {}\n        \n        try:\n            # Test with different model types\n            models = {\n                'RandomForest': RandomForestRegressor(n_estimators=100, random_state=42),\n                'ElasticNet': ElasticNet(random_state=42),\n                'Ridge': Ridge(random_state=42)\n            }\n            \n            for model_name, model in models.items():\n                # Original features\n                model.fit(X_orig_train, y_train)\n                orig_score = model.score(X_orig_test, y_test)\n                \n                # Reduced features\n                model.fit(X_red_train, y_train)\n                red_score = model.score(X_red_test, y_test)\n                \n                metrics[f'{model_name}_original_r2'] = orig_score\n                metrics[f'{model_name}_reduced_r2'] = red_score\n                metrics[f'{model_name}_retention'] = red_score / orig_score if orig_score != 0 else 0\n            \n            # Correlation with target\n            correlations = []\n            for i in range(X_reduced.shape[1]):\n                corr, _ = pearsonr(X_reduced[:, i], y)\n                if not np.isnan(corr):\n                    correlations.append(abs(corr))\n            \n            metrics['avg_correlation'] = np.mean(correlations) if correlations else 0\n            metrics['max_correlation'] = np.max(correlations) if correlations else 0\n            metrics['feature_correlation_std'] = np.std(correlations) if correlations else 0\n            \n            # Dimensionality reduction ratio\n            metrics['compression_ratio'] = X_original.shape[1] / X_reduced.shape[1]\n            \n        except Exception as e:\n            print(f\"Evaluation failed for {method_name}: {str(e)}\")\n            return {}\n        \n        return metrics\n    \n    def apply_linear_methods(self, X, y):\n        \"\"\"Apply linear dimensionality reduction methods\"\"\"\n        \n        print(\"\\n=== LINEAR METHODS ===\")\n        \n        # PCA\n        print(\"Applying PCA...\")\n        pca = PCA(n_components=min(self.target_dims, X.shape[1]))\n        X_pca = pca.fit_transform(X)\n        self.results['PCA'] = X_pca\n        self.fitted_reducers['PCA'] = pca\n        self.evaluation_metrics['PCA'] = self.evaluate_predictive_power(X, X_pca, y, 'PCA')\n        self.evaluation_metrics['PCA']['explained_variance'] = np.sum(pca.explained_variance_ratio_)\n        \n        # Factor Analysis\n        print(\"Applying Factor Analysis...\")\n        try:\n            fa = FactorAnalysis(n_components=min(self.target_dims, X.shape[1]), random_state=42)\n            X_fa = fa.fit_transform(X)\n            self.results['Factor Analysis'] = X_fa\n            self.fitted_reducers['Factor Analysis'] = fa\n            self.evaluation_metrics['Factor Analysis'] = self.evaluate_predictive_power(X, X_fa, y, 'Factor Analysis')\n        except Exception as e:\n            print(f\"Factor Analysis failed: {e}\")\n        \n        # ICA\n        print(\"Applying ICA...\")\n        try:\n            ica = FastICA(n_components=min(self.target_dims, X.shape[1]), random_state=42, max_iter=500)\n            X_ica = ica.fit_transform(X)\n            self.results['ICA'] = X_ica\n            self.fitted_reducers['ICA'] = ica\n            self.evaluation_metrics['ICA'] = self.evaluate_predictive_power(X, X_ica, y, 'ICA')\n        except Exception as e:\n            print(f\"ICA failed: {e}\")\n        \n        # Random Projection\n        print(\"Applying Random Projection...\")\n        rp = GaussianRandomProjection(n_components=self.target_dims, random_state=42)\n        X_rp = rp.fit_transform(X)\n        self.results['Random Projection'] = X_rp\n        self.fitted_reducers['Random Projection'] = rp\n        self.evaluation_metrics['Random Projection'] = self.evaluate_predictive_power(X, X_rp, y, 'Random Projection')\n    \n    def apply_nonlinear_methods(self, X, y):\n        \"\"\"Apply non-linear dimensionality reduction methods\"\"\"\n        \n        print(\"\\n=== NON-LINEAR METHODS ===\")\n        \n        # Kernel PCA\n        print(\"Applying Kernel PCA...\")\n        try:\n            kpca = KernelPCA(n_components=self.target_dims, kernel='rbf', gamma=1.0/X.shape[1])\n            X_kpca = kpca.fit_transform(X)\n            self.results['Kernel PCA'] = X_kpca\n            self.fitted_reducers['Kernel PCA'] = kpca\n            self.evaluation_metrics['Kernel PCA'] = self.evaluate_predictive_power(X, X_kpca, y, 'Kernel PCA')\n        except Exception as e:\n            print(f\"Kernel PCA failed: {e}\")\n        \n        # UMAP\n        print(\"Applying UMAP...\")\n        try:\n            n_neighbors = min(15, len(X) // 4)\n            umap_reducer = umap.UMAP(n_components=self.target_dims, \n                                   n_neighbors=n_neighbors, \n                                   min_dist=0.1, \n                                   random_state=42)\n            X_umap = umap_reducer.fit_transform(X)\n            self.results['UMAP'] = X_umap\n            self.fitted_reducers['UMAP'] = umap_reducer\n            self.evaluation_metrics['UMAP'] = self.evaluate_predictive_power(X, X_umap, y, 'UMAP')\n        except Exception as e:\n            print(f\"UMAP failed: {e}\")\n        \n        # t-SNE (only for smaller datasets due to computational cost)\n        if len(X) <= 5000:\n            print(\"Applying t-SNE...\")\n            try:\n                perplexity = min(30, len(X) // 4)\n                tsne = TSNE(n_components=min(self.target_dims, 3), perplexity=perplexity, random_state=42)\n                X_tsne = tsne.fit_transform(X)\n                self.results['t-SNE'] = X_tsne\n                self.evaluation_metrics['t-SNE'] = self.evaluate_predictive_power(X, X_tsne, y, 't-SNE')\n            except Exception as e:\n                print(f\"t-SNE failed: {e}\")\n        else:\n            print(\"Skipping t-SNE (dataset too large)\")\n        \n        # Isomap\n        print(\"Applying Isomap...\")\n        try:\n            n_neighbors = min(10, len(X) // 10)\n            isomap = Isomap(n_components=self.target_dims, n_neighbors=n_neighbors)\n            X_isomap = isomap.fit_transform(X)\n            self.results['Isomap'] = X_isomap\n            self.fitted_reducers['Isomap'] = isomap\n            self.evaluation_metrics['Isomap'] = self.evaluate_predictive_power(X, X_isomap, y, 'Isomap')\n        except Exception as e:\n            print(f\"Isomap failed: {e}\")\n        \n        # Spectral Embedding\n        print(\"Applying Spectral Embedding...\")\n        try:\n            se = SpectralEmbedding(n_components=self.target_dims, random_state=42)\n            X_se = se.fit_transform(X)\n            self.results['Spectral Embedding'] = X_se\n            self.evaluation_metrics['Spectral Embedding'] = self.evaluate_predictive_power(X, X_se, y, 'Spectral Embedding')\n        except Exception as e:\n            print(f\"Spectral Embedding failed: {e}\")\n    \n    def apply_matrix_factorization(self, X, y):\n        \"\"\"Apply matrix factorization methods\"\"\"\n        \n        print(\"\\n=== MATRIX FACTORIZATION METHODS ===\")\n        \n        # NMF (requires non-negative data)\n        print(\"Applying NMF...\")\n        try:\n            X_positive = MinMaxScaler().fit_transform(X)\n            nmf = NMF(n_components=self.target_dims, random_state=42, max_iter=500)\n            X_nmf = nmf.fit_transform(X_positive)\n            self.results['NMF'] = X_nmf\n            self.fitted_reducers['NMF'] = nmf\n            self.evaluation_metrics['NMF'] = self.evaluate_predictive_power(X, X_nmf, y, 'NMF')\n        except Exception as e:\n            print(f\"NMF failed: {e}\")\n        \n        # Dictionary Learning\n        print(\"Applying Dictionary Learning...\")\n        try:\n            dict_learning = DictionaryLearning(n_components=self.target_dims, random_state=42, max_iter=100)\n            X_dict = dict_learning.fit_transform(X)\n            self.results['Dictionary Learning'] = X_dict\n            self.fitted_reducers['Dictionary Learning'] = dict_learning\n            self.evaluation_metrics['Dictionary Learning'] = self.evaluate_predictive_power(X, X_dict, y, 'Dictionary Learning')\n        except Exception as e:\n            print(f\"Dictionary Learning failed: {e}\")\n    \n    def apply_autoencoder_methods(self, X, y):\n        \"\"\"Apply autoencoder-based methods\"\"\"\n        \n        if not TENSORFLOW_AVAILABLE:\n            print(\"Skipping autoencoder methods (TensorFlow not available)\")\n            return\n        \n        print(\"\\n=== AUTOENCODER METHODS ===\")\n        \n        def build_autoencoder(input_dim, encoding_dim, hidden_layers=[128, 64]):\n            encoder_input = layers.Input(shape=(input_dim,))\n            x = encoder_input\n            \n            for units in hidden_layers:\n                x = layers.Dense(units, activation='relu')(x)\n                x = layers.BatchNormalization()(x)\n                x = layers.Dropout(0.2)(x)\n            \n            encoded = layers.Dense(encoding_dim, activation='linear', name='encoded')(x)\n            \n            # Decoder\n            x = encoded\n            for units in reversed(hidden_layers):\n                x = layers.Dense(units, activation='relu')(x)\n                x = layers.BatchNormalization()(x)\n                x = layers.Dropout(0.2)(x)\n            \n            decoded = layers.Dense(input_dim, activation='linear')(x)\n            \n            autoencoder = Model(encoder_input, decoded)\n            encoder = Model(encoder_input, encoded)\n            \n            return autoencoder, encoder\n        \n        # Standard Autoencoder\n        print(\"Applying Autoencoder...\")\n        try:\n            autoencoder, encoder = build_autoencoder(X.shape[1], self.target_dims)\n            autoencoder.compile(optimizer='adam', loss='mse')\n            autoencoder.fit(X, X, epochs=50, batch_size=64, verbose=0, validation_split=0.2)\n            X_ae = encoder.predict(X)\n            self.results['Autoencoder'] = X_ae\n            self.fitted_reducers['Autoencoder'] = encoder\n            self.evaluation_metrics['Autoencoder'] = self.evaluate_predictive_power(X, X_ae, y, 'Autoencoder')\n        except Exception as e:\n            print(f\"Autoencoder failed: {e}\")\n        \n        # Sparse Autoencoder\n        print(\"Applying Sparse Autoencoder...\")\n        try:\n            sparse_input = layers.Input(shape=(X.shape[1],))\n            x = layers.Dense(128, activation='relu')(sparse_input)\n            x = layers.Dense(64, activation='relu')(x)\n            sparse_encoded = layers.Dense(self.target_dims * 2, activation='relu',\n                                        activity_regularizer=regularizers.l1(1e-5))(x)\n            final_encoded = layers.Dense(self.target_dims, activation='linear')(sparse_encoded)\n            \n            x = layers.Dense(self.target_dims * 2, activation='relu')(final_encoded)\n            x = layers.Dense(64, activation='relu')(x)\n            x = layers.Dense(128, activation='relu')(x)\n            decoded = layers.Dense(X.shape[1], activation='linear')(x)\n            \n            sparse_ae = Model(sparse_input, decoded)\n            sparse_encoder = Model(sparse_input, final_encoded)\n            \n            sparse_ae.compile(optimizer='adam', loss='mse')\n            sparse_ae.fit(X, X, epochs=50, batch_size=64, verbose=0, validation_split=0.2)\n            X_sparse = sparse_encoder.predict(X)\n            self.results['Sparse Autoencoder'] = X_sparse\n            self.fitted_reducers['Sparse Autoencoder'] = sparse_encoder\n            self.evaluation_metrics['Sparse Autoencoder'] = self.evaluate_predictive_power(X, X_sparse, y, 'Sparse Autoencoder')\n        except Exception as e:\n            print(f\"Sparse Autoencoder failed: {e}\")\n    \n    def apply_ensemble_methods(self, X, y):\n        \"\"\"Apply ensemble and hybrid methods\"\"\"\n        \n        print(\"\\n=== ENSEMBLE METHODS ===\")\n        \n        # Ensemble of top performing methods\n        if len(self.results) >= 3:\n            print(\"Creating ensemble from top methods...\")\n            \n            # Get top 3 methods by average correlation\n            method_scores = {}\n            for method, metrics in self.evaluation_metrics.items():\n                if 'avg_correlation' in metrics:\n                    method_scores[method] = metrics['avg_correlation']\n            \n            if method_scores:\n                top_methods = sorted(method_scores.items(), key=lambda x: x[1], reverse=True)[:3]\n                \n                embeddings = []\n                for method_name, _ in top_methods:\n                    if method_name in self.results:\n                        embedding = self.results[method_name]\n                        # Normalize to same scale\n                        embedding_norm = StandardScaler().fit_transform(embedding)\n                        embeddings.append(embedding_norm)\n                \n                if len(embeddings) >= 2:\n                    # Average ensemble\n                    X_ensemble_avg = np.mean(embeddings, axis=0)\n                    self.results['Ensemble (Average)'] = X_ensemble_avg\n                    self.evaluation_metrics['Ensemble (Average)'] = self.evaluate_predictive_power(X, X_ensemble_avg, y, 'Ensemble (Average)')\n                    \n                    # Concatenated ensemble\n                    X_ensemble_concat = np.hstack(embeddings)\n                    if X_ensemble_concat.shape[1] > self.target_dims:\n                        # Reduce with PCA\n                        pca_final = PCA(n_components=self.target_dims)\n                        X_ensemble_concat = pca_final.fit_transform(X_ensemble_concat)\n                    \n                    self.results['Ensemble (Concatenated)'] = X_ensemble_concat\n                    self.evaluation_metrics['Ensemble (Concatenated)'] = self.evaluate_predictive_power(X, X_ensemble_concat, y, 'Ensemble (Concatenated)')\n        \n        # Two-stage reduction\n        print(\"Applying two-stage reduction...\")\n        try:\n            # Stage 1: Fast pre-reduction with Random Projection\n            rp_stage1 = GaussianRandomProjection(n_components=min(200, X.shape[1]//2), random_state=42)\n            X_stage1 = rp_stage1.fit_transform(X)\n            \n            # Stage 2: UMAP on reduced space\n            umap_stage2 = umap.UMAP(n_components=self.target_dims, random_state=42)\n            X_two_stage = umap_stage2.fit_transform(X_stage1)\n            \n            self.results['Two-Stage (RP+UMAP)'] = X_two_stage\n            self.evaluation_metrics['Two-Stage (RP+UMAP)'] = self.evaluate_predictive_power(X, X_two_stage, y, 'Two-Stage (RP+UMAP)')\n        except Exception as e:\n            print(f\"Two-stage reduction failed: {e}\")\n    \n    def run_comprehensive_analysis(self, X, y):\n        \"\"\"Run all dimensionality reduction methods\"\"\"\n        \n        print(f\"Starting comprehensive analysis on {X.shape[0]} samples with {X.shape[1]} features\")\n        print(f\"Target dimensions: {self.target_dims}\")\n        \n        # Apply all methods\n        self.apply_linear_methods(X, y)\n        self.apply_nonlinear_methods(X, y)\n        self.apply_matrix_factorization(X, y)\n        self.apply_autoencoder_methods(X, y)\n        self.apply_ensemble_methods(X, y)\n        \n        return self.results, self.evaluation_metrics\n    \n    def create_comprehensive_visualizations(self, y):\n        \"\"\"Create comprehensive visualizations of results\"\"\"\n        \n        print(\"\\n=== CREATING VISUALIZATIONS ===\")\n        \n        # 1. Performance comparison\n        if self.evaluation_metrics:\n            metrics_df = pd.DataFrame(self.evaluation_metrics).T\n            \n            # Plot average correlations\n            if 'avg_correlation' in metrics_df.columns:\n                plt.figure(figsize=(15, 8))\n                metrics_df = metrics_df.sort_values('avg_correlation', ascending=True)\n                plt.barh(range(len(metrics_df)), metrics_df['avg_correlation'])\n                plt.yticks(range(len(metrics_df)), metrics_df.index)\n                plt.xlabel('Average Correlation with Target')\n                plt.title('Dimensionality Reduction Methods: Average Correlation with Target')\n                plt.tight_layout()\n                plt.show()\n            \n            # Plot compression vs performance\n            if 'compression_ratio' in metrics_df.columns and 'avg_correlation' in metrics_df.columns:\n                plt.figure(figsize=(12, 8))\n                plt.scatter(metrics_df['compression_ratio'], metrics_df['avg_correlation'], \n                           s=100, alpha=0.7)\n                \n                for i, method in enumerate(metrics_df.index):\n                    plt.annotate(method, \n                               (metrics_df.loc[method, 'compression_ratio'], \n                                metrics_df.loc[method, 'avg_correlation']),\n                               xytext=(5, 5), textcoords='offset points', fontsize=8)\n                \n                plt.xlabel('Compression Ratio (Original Dims / Reduced Dims)')\n                plt.ylabel('Average Correlation with Target')\n                plt.title('Compression vs Performance Trade-off')\n                plt.grid(True, alpha=0.3)\n                plt.tight_layout()\n                plt.show()\n        \n        # 2. Sample embeddings (first 2 dimensions)\n        if self.results:\n            n_methods = min(9, len(self.results))  # Show max 9 methods\n            fig, axes = plt.subplots(3, 3, figsize=(15, 15))\n            axes = axes.ravel()\n            \n            for i, (method, embedding) in enumerate(list(self.results.items())[:n_methods]):\n                ax = axes[i]\n                scatter = ax.scatter(embedding[:, 0], embedding[:, 1], \n                                   c=y, cmap='viridis', alpha=0.6, s=20)\n                ax.set_title(f'{method}')\n                ax.set_xlabel('Dimension 1')\n                ax.set_ylabel('Dimension 2')\n                plt.colorbar(scatter, ax=ax)\n            \n            # Hide unused subplots\n            for i in range(n_methods, 9):\n                axes[i].set_visible(False)\n            \n            plt.suptitle('Dimensionality Reduction Results (First 2 Dimensions)', fontsize=16)\n            plt.tight_layout()\n            plt.show()\n        \n        return metrics_df if self.evaluation_metrics else None\n\n\ndef main():\n    \"\"\"Main analysis function\"\"\"\n    \n    print(\"=\"*80)\n    print(\"COMPREHENSIVE DRW CRYPTO DIMENSIONALITY REDUCTION ANALYSIS\")\n    print(\"=\"*80)\n    \n    # Load data\n    print(\"\\nLoading DRW crypto dataset...\")\n    try:\n        # Load the last 10,000 rows\n        df = pd.read_parquet('train.parquet')\n        df_last = df.tail(20000).copy()\n        \n        print(f\"Loaded {len(df_last)} samples with {df_last.shape[1]} columns\")\n        print(f\"Dataset date range: {df_last['timestamp'].min()} to {df_last['timestamp'].max()}\")\n        \n        # Separate features and target\n        feature_cols = [col for col in df_last.columns if col not in ['timestamp', 'label']]\n        X = df_last[feature_cols].values\n        y = df_last['label'].values\n        \n        print(f\"Features: {len(feature_cols)} ({feature_cols[:5]}... + {len(feature_cols)-5} more)\")\n        print(f\"Target stats: mean={y.mean():.6f}, std={y.std():.6f}, range=[{y.min():.6f}, {y.max():.6f}]\")\n        \n    except FileNotFoundError:\n        print(\"train.parquet not found. Creating sample data for demonstration...\")\n        # Create realistic crypto-like data for demonstration\n        np.random.seed(42)\n        n_samples, n_features = 20000, 895\n        \n        # Generate features with different characteristics\n        X_normal = np.random.normal(0, 1, (n_samples, n_features//3))\n        X_skewed = np.random.lognormal(0, 1, (n_samples, n_features//3))\n        X_sparse = np.random.exponential(0.1, (n_samples, n_features - 2*(n_features//3)))\n        \n        # Add some infinite values and outliers to simulate real financial data\n        X_sparse[np.random.random((n_samples, X_sparse.shape[1])) < 0.001] = np.inf\n        X_sparse[np.random.random((n_samples, X_sparse.shape[1])) < 0.001] = -np.inf\n        \n        X = np.hstack([X_normal, X_skewed, X_sparse])\n        \n        # Generate target with some correlation to features\n        y = (np.sum(X_normal[:, :10], axis=1) + \n             np.log1p(np.sum(np.abs(X_skewed[:, :10]), axis=1)) + \n             np.random.normal(0, 0.1, n_samples))\n        \n        print(f\"Generated sample data: {X.shape} features, {len(y)} samples\")\n    \n    # Data preprocessing\n    print(\"\\n\" + \"=\"*50)\n    print(\"DATA PREPROCESSING\")\n    print(\"=\"*50)\n    \n    preprocessor = CryptoDataPreprocessor()\n    X_processed, mask = preprocessor.fit_transform(X, y)\n    \n    if mask is not None:\n        y_processed = y[mask]\n        print(f\"Removed {len(y) - len(y_processed)} samples with infinite values\")\n    else:\n        y_processed = y\n    \n    print(f\"Final preprocessed data: {X_processed.shape}\")\n    print(f\"Features with variance > 0: {X_processed.shape[1]}\")\n    \n    # Check for any remaining issues\n    print(f\"Infinite values: {np.isinf(X_processed).sum()}\")\n    print(f\"NaN values: {np.isnan(X_processed).sum()}\")\n    print(f\"Feature value ranges: [{np.min(X_processed):.3f}, {np.max(X_processed):.3f}]\")\n    \n    # Dimensionality reduction analysis\n    print(\"\\n\" + \"=\"*50)\n    print(\"DIMENSIONALITY REDUCTION ANALYSIS\")\n    print(\"=\"*50)\n    \n    # Test different target dimensions\n    target_dimensions = [25, 50, 100]\n    all_results = {}\n    \n    for target_dim in target_dimensions:\n        print(f\"\\n--- Analysis with {target_dim} target dimensions ---\")\n        \n        evaluator = DimensionalityReductionEvaluator(target_dims=target_dim)\n        results, metrics = evaluator.run_comprehensive_analysis(X_processed, y_processed)\n        \n        all_results[target_dim] = {\n            'results': results,\n            'metrics': metrics,\n            'evaluator': evaluator\n        }\n    \n    # Comprehensive analysis and recommendations\n    print(\"\\n\" + \"=\"*80)\n    print(\"ANALYSIS RESULTS AND RECOMMENDATIONS\")\n    print(\"=\"*80)\n    \n    # Use 50 dimensions for detailed analysis\n    main_results = all_results[50]\n    evaluator = main_results['evaluator']\n    \n    # Create visualizations\n    metrics_df = evaluator.create_comprehensive_visualizations(y_processed)\n    \n    # Print detailed results\n    if metrics_df is not None:\n        print(\"\\n=== DETAILED PERFORMANCE METRICS ===\")\n        \n        # Sort by average correlation\n        metrics_df = metrics_df.sort_values('avg_correlation', ascending=False)\n        \n        print(\"\\nTop Methods by Average Correlation with Target:\")\n        for method in metrics_df.index[:10]:\n            corr = metrics_df.loc[method, 'avg_correlation']\n            if 'RandomForest_retention' in metrics_df.columns:\n                retention = metrics_df.loc[method, 'RandomForest_retention']\n                print(f\"{method:25s}: Correlation={corr:.4f}, RF Retention={retention:.3f}\")\n            else:\n                print(f\"{method:25s}: Correlation={corr:.4f}\")\n        \n        # Model-specific recommendations\n        print(\"\\n=== RECOMMENDATIONS BY DOWNSTREAM MODEL TYPE ===\")\n        \n        model_recommendations = {\n            'Tree-based models (RF, XGB, LightGBM)': [],\n            'Linear models (Ridge, Lasso, ElasticNet)': [],\n            'Neural networks': [],\n            'SVM/Kernel methods': []\n        }\n        \n        for method in metrics_df.index:\n            corr = metrics_df.loc[method, 'avg_correlation']\n            if corr > 0.1:  # Only recommend methods with decent correlation\n                if 'PCA' in method or 'Random' in method:\n                    model_recommendations['Linear models (Ridge, Lasso, ElasticNet)'].append(\n                        (method, corr))\n                elif 'Autoencoder' in method or 'VAE' in method:\n                    model_recommendations['Neural networks'].append((method, corr))\n                elif 'Kernel' in method or 'UMAP' in method:\n                    model_recommendations['SVM/Kernel methods'].append((method, corr))\n                else:\n                    model_recommendations['Tree-based models (RF, XGB, LightGBM)'].append(\n                        (method, corr))\n        \n        for model_type, methods in model_recommendations.items():\n            if methods:\n                print(f\"\\n{model_type}:\")\n                for method, corr in sorted(methods, key=lambda x: x[1], reverse=True)[:3]:\n                    print(f\"  - {method} (correlation: {corr:.4f})\")\n        \n        # Computational efficiency analysis\n        print(\"\\n=== COMPUTATIONAL EFFICIENCY RECOMMENDATIONS ===\")\n        \n        fast_methods = ['PCA', 'Random Projection', 'Factor Analysis']\n        medium_methods = ['UMAP', 'Kernel PCA', 'NMF']\n        slow_methods = ['t-SNE', 'Autoencoder', 'Isomap']\n        \n        print(\"\\nFor FAST processing (real-time inference):\")\n        for method in fast_methods:\n            if method in metrics_df.index:\n                corr = metrics_df.loc[method, 'avg_correlation']\n                print(f\"  - {method}: {corr:.4f} correlation\")\n        \n        print(\"\\nFor BALANCED speed/quality:\")\n        for method in medium_methods:\n            if method in metrics_df.index:\n                corr = metrics_df.loc[method, 'avg_correlation']\n                print(f\"  - {method}: {corr:.4f} correlation\")\n        \n        print(\"\\nFor MAXIMUM quality (batch processing):\")\n        best_methods = metrics_df.head(5).index\n        for method in best_methods:\n            corr = metrics_df.loc[method, 'avg_correlation']\n            print(f\"  - {method}: {corr:.4f} correlation\")\n    \n    # Final recommendations\n    print(\"\\n\" + \"=\"*80)\n    print(\"FINAL RECOMMENDATIONS FOR DRW CRYPTO COMPETITION\")\n    print(\"=\"*80)\n    \n    print(\"\"\"\n    STRATEGIC RECOMMENDATIONS:\n    \n    1. **PRIMARY APPROACH (Highest Performance):**\n       - Use ensemble of top 3 methods identified above\n       - Target 50-100 dimensions for optimal balance\n       - Validate on hold-out set before submission\n    \n    2. **COMPUTATIONAL CONSTRAINTS:**\n       - For speed: Random Projection + PCA pipeline\n       - For quality: UMAP or best-performing autoencoder\n       - For robustness: Ensemble of complementary methods\n    \n    3. **PREPROCESSING STRATEGY:**\n       - Use RobustScaler for outlier handling\n       - Remove zero-variance features\n       - Consider feature selection before reduction\n    \n    4. **VALIDATION APPROACH:**\n       - Use Pearson correlation as primary metric (matches competition)\n       - Cross-validate with multiple model types\n       - Check temporal stability if using time-based splits\n    \n    5. **IMPLEMENTATION TIPS:**\n       - Save all fitted transformers for test set\n       - Use consistent random seeds for reproducibility\n       - Monitor for overfitting in reduction process\n       - Consider different reductions for different feature groups\n    \"\"\")\n    \n    return all_results, metrics_df\n\n\nif __name__ == \"__main__\":\n    results, metrics = main()\n    print(\"\\n✅ Comprehensive analysis complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-17T01:54:38.789665Z","iopub.execute_input":"2025-06-17T01:54:38.790109Z","iopub.status.idle":"2025-06-17T03:10:46.043413Z","shell.execute_reply.started":"2025-06-17T01:54:38.79009Z","shell.execute_reply":"2025-06-17T03:10:46.042317Z"}},"outputs":[],"execution_count":null}]}