{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python3\n\"\"\"\nDRW Crypto Market Prediction - Comprehensive Dimensionality Reduction Analysis\n============================================================================\nTesting 25+ dimensionality reduction techniques on rank-transformed financial data\n\"\"\"\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom scipy import stats\nimport warnings\nimport os\nimport gc\nimport time\nfrom datetime import datetime\nfrom tqdm import tqdm\nwarnings.filterwarnings('ignore')\n\n# Import all dimensionality reduction methods\nfrom sklearn.decomposition import (\n    PCA, FastICA, NMF, FactorAnalysis, TruncatedSVD,\n    SparsePCA, MiniBatchSparsePCA, KernelPCA,\n    IncrementalPCA, DictionaryLearning\n)\nfrom sklearn.manifold import (\n    TSNE, Isomap, LocallyLinearEmbedding, \n    SpectralEmbedding, MDS\n)\nfrom sklearn.feature_selection import (\n    SelectKBest, f_regression, mutual_info_regression,\n    RFE, SelectFromModel, VarianceThreshold,\n    chi2, SelectPercentile\n)\nfrom sklearn.random_projection import (\n    GaussianRandomProjection, SparseRandomProjection\n)\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.cross_decomposition import PLSRegression, CCA\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\nfrom sklearn.linear_model import LassoCV, ElasticNetCV\nimport xgboost as xgb\nimport lightgbm as lgb\n\n# Try importing additional methods\ntry:\n    import umap\n    UMAP_AVAILABLE = True\nexcept ImportError:\n    UMAP_AVAILABLE = False\n    print(\"UMAP not available. Install with: pip install umap-learn\")\n\ntry:\n    from sklearn.neural_network import MLPRegressor\n    NEURAL_AVAILABLE = True\nexcept ImportError:\n    NEURAL_AVAILABLE = False\n\n# Set random seed\nnp.random.seed(42)\n\n# Check if we're in Kaggle environment\nKAGGLE_INPUT_PATH = '/kaggle/input/drw-crypto-market-prediction'\nif os.path.exists(KAGGLE_INPUT_PATH):\n    DATA_PATH = KAGGLE_INPUT_PATH\nelse:\n    DATA_PATH = '.'\n\nclass FinancialDimensionalityReducer:\n    \"\"\"\n    Comprehensive dimensionality reduction for financial data\n    \"\"\"\n    \n    def __init__(self, n_components=50, sample_size=10000, n_jobs=4):\n        self.n_components = n_components\n        self.sample_size = sample_size\n        self.n_jobs = n_jobs\n        self.results = {}\n        self.best_method = None\n        self.reducers = {}\n        \n    def optimize_dtypes(self, df):\n        \"\"\"Optimize dataframe dtypes to save memory\"\"\"\n        for col in df.columns:\n            col_type = df[col].dtype\n            \n            if col_type != 'object':\n                c_min = df[col].min()\n                c_max = df[col].max()\n                \n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                else:\n                    df[col] = df[col].astype(np.float32)\n        \n        return df\n    \n    def handle_infinities(self, X):\n        \"\"\"Replace inf/-inf with appropriate finite values\"\"\"\n        X = X.copy()\n        \n        for i in range(X.shape[1]):\n            col = X[:, i]\n            finite_mask = np.isfinite(col)\n            \n            if np.any(~finite_mask):\n                finite_vals = col[finite_mask]\n                \n                if len(finite_vals) > 0:\n                    # Replace inf with max + 1 std, -inf with min - 1 std\n                    col_max = np.max(finite_vals)\n                    col_min = np.min(finite_vals)\n                    col_std = np.std(finite_vals)\n                    \n                    col[col == np.inf] = col_max + col_std\n                    col[col == -np.inf] = col_min - col_std\n                    \n                    # Replace any remaining non-finite with median\n                    col[~np.isfinite(col)] = np.median(finite_vals)\n                else:\n                    # If all values are non-finite, replace with 0\n                    col[:] = 0\n        \n        return X\n    \n    def remove_constant_features(self, X, feature_names, variance_threshold=1e-10):\n        \"\"\"Remove features with near-zero variance\"\"\"\n        print(f\"Removing constant features (variance < {variance_threshold})...\")\n        \n        valid_indices = []\n        \n        for i in range(X.shape[1]):\n            col = X[:, i]\n            finite_mask = np.isfinite(col)\n            \n            if np.any(finite_mask):\n                finite_vals = col[finite_mask]\n                if len(finite_vals) > 1 and np.var(finite_vals) > variance_threshold:\n                    valid_indices.append(i)\n        \n        valid_indices = np.array(valid_indices)\n        print(f\"Kept {len(valid_indices)} / {X.shape[1]} features\")\n        \n        return X[:, valid_indices], np.array(feature_names)[valid_indices]\n    \n    def rank_transform(self, X):\n        \"\"\"\n        Apply rank transformation - best practice for financial data\n        Converts each feature to its percentile rank\n        \"\"\"\n        print(\"Applying rank transformation...\")\n        X_ranked = np.zeros_like(X)\n        \n        for i in range(X.shape[1]):\n            col = X[:, i]\n            finite_mask = np.isfinite(col)\n            \n            if np.any(finite_mask):\n                # Rank transform only finite values\n                finite_vals = col[finite_mask]\n                ranks = stats.rankdata(finite_vals, method='average')\n                # Convert to percentiles (0-1 range)\n                percentiles = (ranks - 1) / (len(ranks) - 1) if len(ranks) > 1 else ranks\n                \n                # Fill in the ranked values\n                X_ranked[finite_mask, i] = percentiles\n                # Non-finite values get 0.5 (median rank)\n                X_ranked[~finite_mask, i] = 0.5\n            else:\n                # If all values are non-finite, use 0.5\n                X_ranked[:, i] = 0.5\n        \n        return X_ranked.astype(np.float32)\n    \n    def get_dimensionality_reducers(self):\n        \"\"\"Get all dimensionality reduction methods to test\"\"\"\n        reducers = {}\n        \n        # 1. Linear decomposition methods\n        reducers['PCA'] = PCA(n_components=self.n_components, random_state=42)\n        reducers['IncrementalPCA'] = IncrementalPCA(n_components=self.n_components, batch_size=1000)\n        reducers['TruncatedSVD'] = TruncatedSVD(n_components=self.n_components, random_state=42)\n        reducers['FastICA'] = FastICA(n_components=min(self.n_components, 50), random_state=42, max_iter=500)\n        reducers['FactorAnalysis'] = FactorAnalysis(n_components=self.n_components, random_state=42)\n        \n        # 2. Sparse methods\n        reducers['SparsePCA'] = SparsePCA(n_components=20, random_state=42, max_iter=100)\n        reducers['MiniBatchSparsePCA'] = MiniBatchSparsePCA(n_components=20, random_state=42, batch_size=100)\n        reducers['DictionaryLearning'] = DictionaryLearning(n_components=30, random_state=42, max_iter=100)\n        \n        # 3. Non-negative matrix factorization (for rank-transformed data)\n        reducers['NMF'] = NMF(n_components=self.n_components, random_state=42, max_iter=500)\n        \n        # 4. Kernel methods\n        reducers['KernelPCA_rbf'] = KernelPCA(n_components=self.n_components, kernel='rbf', gamma=0.01, random_state=42)\n        reducers['KernelPCA_sigmoid'] = KernelPCA(n_components=self.n_components, kernel='sigmoid', random_state=42)\n        \n        # 5. Random projections\n        reducers['GaussianRandomProjection'] = GaussianRandomProjection(n_components=self.n_components, random_state=42)\n        reducers['SparseRandomProjection'] = SparseRandomProjection(n_components=self.n_components, random_state=42)\n        \n        # 6. Manifold learning (use fewer components due to computational cost)\n        manifold_components = min(10, self.n_components)\n        reducers['Isomap'] = Isomap(n_components=manifold_components, n_neighbors=10)\n        reducers['LocallyLinearEmbedding'] = LocallyLinearEmbedding(n_components=manifold_components, n_neighbors=10, random_state=42)\n        reducers['SpectralEmbedding'] = SpectralEmbedding(n_components=manifold_components, random_state=42)\n        reducers['MDS'] = MDS(n_components=manifold_components, random_state=42, max_iter=100)\n        \n        # Only include t-SNE for very small samples due to computational cost\n        if self.sample_size <= 5000:\n            reducers['TSNE'] = TSNE(n_components=2, random_state=42, perplexity=30)\n        \n        # 7. UMAP if available\n        if UMAP_AVAILABLE:\n            reducers['UMAP'] = umap.UMAP(n_components=manifold_components, random_state=42)\n        \n        # 8. Supervised methods (these need the target variable)\n        # Will be handled separately\n        \n        # 9. Feature selection methods\n        # Will be handled separately\n        \n        return reducers\n    \n    def get_feature_selection_methods(self, X, y):\n        \"\"\"Get feature selection methods (these need target variable)\"\"\"\n        selectors = {}\n        \n        # Statistical methods\n        selectors['SelectKBest_f_regression'] = SelectKBest(f_regression, k=self.n_components)\n        selectors['SelectKBest_mutual_info'] = SelectKBest(mutual_info_regression, k=self.n_components)\n        selectors['SelectPercentile_f_regression'] = SelectPercentile(f_regression, percentile=20)\n        \n        # Variance threshold\n        selectors['VarianceThreshold_0.01'] = VarianceThreshold(threshold=0.01)\n        selectors['VarianceThreshold_0.05'] = VarianceThreshold(threshold=0.05)\n        \n        # Model-based selection\n        selectors['L1_Lasso'] = SelectFromModel(LassoCV(cv=3, random_state=42, max_iter=1000), threshold='median')\n        selectors['L1_ElasticNet'] = SelectFromModel(ElasticNetCV(cv=3, random_state=42, max_iter=1000), threshold='median')\n        \n        # Tree-based selection\n        selectors['RandomForest'] = SelectFromModel(\n            RandomForestRegressor(n_estimators=50, random_state=42, n_jobs=self.n_jobs),\n            threshold='median'\n        )\n        selectors['ExtraTrees'] = SelectFromModel(\n            ExtraTreesRegressor(n_estimators=50, random_state=42, n_jobs=self.n_jobs),\n            threshold='median'\n        )\n        \n        # Recursive feature elimination (use faster models)\n        if X.shape[1] > 100:\n            # For many features, use simpler models in RFE\n            base_estimator = xgb.XGBRegressor(n_estimators=50, max_depth=3, random_state=42)\n            selectors['RFE_XGBoost'] = RFE(base_estimator, n_features_to_select=self.n_components, step=10)\n        \n        return selectors\n    \n    def get_supervised_methods(self):\n        \"\"\"Get supervised dimensionality reduction methods\"\"\"\n        supervised = {}\n        \n        # PLS methods\n        supervised['PLSRegression'] = PLSRegression(n_components=min(self.n_components, 20))\n        \n        # CCA (Canonical Correlation Analysis)\n        supervised['CCA'] = CCA(n_components=min(self.n_components, 20))\n        \n        return supervised\n    \n    def evaluate_method(self, X_train_reduced, X_val_reduced, y_train, y_val, method_name):\n        \"\"\"Evaluate a dimensionality reduction method using XGBoost\"\"\"\n        # Train a simple XGBoost model\n        model = xgb.XGBRegressor(\n            n_estimators=100,\n            max_depth=4,\n            learning_rate=0.1,\n            random_state=42,\n            n_jobs=self.n_jobs\n        )\n        \n        # Handle edge cases\n        if X_train_reduced.shape[1] == 0:\n            return {\n                'method': method_name,\n                'n_features': 0,\n                'rmse': np.inf,\n                'r2': -np.inf,\n                'pearson': 0,\n                'train_time': 0,\n                'error': 'No features selected'\n            }\n        \n        try:\n            start_time = time.time()\n            model.fit(X_train_reduced, y_train)\n            train_time = time.time() - start_time\n            \n            y_pred = model.predict(X_val_reduced)\n            \n            rmse = np.sqrt(mean_squared_error(y_val, y_pred))\n            r2 = r2_score(y_val, y_pred)\n            pearson_corr, _ = stats.pearsonr(y_val, y_pred)\n            \n            return {\n                'method': method_name,\n                'n_features': X_train_reduced.shape[1],\n                'rmse': rmse,\n                'r2': r2,\n                'pearson': pearson_corr,\n                'train_time': train_time,\n                'error': None\n            }\n        except Exception as e:\n            return {\n                'method': method_name,\n                'n_features': X_train_reduced.shape[1],\n                'rmse': np.inf,\n                'r2': -np.inf,\n                'pearson': 0,\n                'train_time': 0,\n                'error': str(e)\n            }\n    \n    def run_experiments(self, df, feature_cols):\n        \"\"\"Run all dimensionality reduction experiments\"\"\"\n        print(\"=\"*60)\n        print(\"FINANCIAL DIMENSIONALITY REDUCTION EXPERIMENTS\")\n        print(f\"Sample size: {self.sample_size}\")\n        print(f\"Target dimensions: {self.n_components}\")\n        print(\"=\"*60)\n        \n        # Sample data\n        if len(df) > self.sample_size:\n            # Use most recent samples\n            df_sample = df.iloc[-self.sample_size:].copy()\n        else:\n            df_sample = df.copy()\n        \n        print(f\"\\nUsing {len(df_sample)} samples\")\n        \n        # Extract features and target\n        X = df_sample[feature_cols].values.astype(np.float32)\n        y = df_sample['label'].values.astype(np.float32)\n        \n        # Clean data\n        print(\"\\n1. Data preprocessing...\")\n        X = self.handle_infinities(X)\n        X, feature_names = self.remove_constant_features(X, feature_cols)\n        \n        # Rank transform - crucial for financial data\n        X_ranked = self.rank_transform(X)\n        print(f\"Data shape after preprocessing: {X_ranked.shape}\")\n        \n        # Split data\n        X_train, X_val, y_train, y_val = train_test_split(\n            X_ranked, y, test_size=0.2, random_state=42\n        )\n        \n        # Results storage\n        all_results = []\n        \n        print(\"\\n2. Testing unsupervised methods...\")\n        print(\"-\" * 40)\n        \n        # Test unsupervised methods\n        reducers = self.get_dimensionality_reducers()\n        \n        for name, reducer in tqdm(reducers.items(), desc=\"Unsupervised methods\"):\n            try:\n                # Fit and transform\n                start_time = time.time()\n                X_train_reduced = reducer.fit_transform(X_train)\n                X_val_reduced = reducer.transform(X_val)\n                reduction_time = time.time() - start_time\n                \n                # Evaluate\n                result = self.evaluate_method(X_train_reduced, X_val_reduced, y_train, y_val, name)\n                result['reduction_time'] = reduction_time\n                result['method_type'] = 'unsupervised'\n                all_results.append(result)\n                \n                # Store reducer for later use\n                self.reducers[name] = reducer\n                \n            except Exception as e:\n                print(f\"Error with {name}: {str(e)}\")\n                all_results.append({\n                    'method': name,\n                    'method_type': 'unsupervised',\n                    'error': str(e),\n                    'rmse': np.inf,\n                    'r2': -np.inf\n                })\n        \n        print(\"\\n3. Testing feature selection methods...\")\n        print(\"-\" * 40)\n        \n        # Test feature selection methods\n        selectors = self.get_feature_selection_methods(X_train, y_train)\n        \n        for name, selector in tqdm(selectors.items(), desc=\"Feature selection\"):\n            try:\n                # Fit and transform\n                start_time = time.time()\n                X_train_reduced = selector.fit_transform(X_train, y_train)\n                X_val_reduced = selector.transform(X_val)\n                reduction_time = time.time() - start_time\n                \n                # For feature selection, we can identify which features were selected\n                if hasattr(selector, 'get_support'):\n                    selected_features = feature_names[selector.get_support()]\n                else:\n                    selected_features = None\n                \n                # Evaluate\n                result = self.evaluate_method(X_train_reduced, X_val_reduced, y_train, y_val, name)\n                result['reduction_time'] = reduction_time\n                result['method_type'] = 'feature_selection'\n                result['selected_features'] = selected_features\n                all_results.append(result)\n                \n                # Store selector\n                self.reducers[name] = selector\n                \n            except Exception as e:\n                print(f\"Error with {name}: {str(e)}\")\n                all_results.append({\n                    'method': name,\n                    'method_type': 'feature_selection',\n                    'error': str(e),\n                    'rmse': np.inf,\n                    'r2': -np.inf\n                })\n        \n        print(\"\\n4. Testing supervised methods...\")\n        print(\"-\" * 40)\n        \n        # Test supervised methods\n        supervised_methods = self.get_supervised_methods()\n        \n        for name, method in tqdm(supervised_methods.items(), desc=\"Supervised methods\"):\n            try:\n                # Fit and transform\n                start_time = time.time()\n                method.fit(X_train, y_train)\n                X_train_reduced = method.transform(X_train)\n                X_val_reduced = method.transform(X_val)\n                reduction_time = time.time() - start_time\n                \n                # Evaluate\n                result = self.evaluate_method(X_train_reduced, X_val_reduced, y_train, y_val, name)\n                result['reduction_time'] = reduction_time\n                result['method_type'] = 'supervised'\n                all_results.append(result)\n                \n                # Store method\n                self.reducers[name] = method\n                \n            except Exception as e:\n                print(f\"Error with {name}: {str(e)}\")\n                all_results.append({\n                    'method': name,\n                    'method_type': 'supervised',\n                    'error': str(e),\n                    'rmse': np.inf,\n                    'r2': -np.inf\n                })\n        \n        # Convert to DataFrame and sort by performance\n        self.results = pd.DataFrame(all_results)\n        self.results = self.results[self.results['error'].isna()].sort_values('pearson', ascending=False)\n        \n        # Identify best method\n        if len(self.results) > 0:\n            self.best_method = self.results.iloc[0]['method']\n        \n        return self.results\n    \n    def create_comprehensive_visualization(self):\n        \"\"\"Create comprehensive visualization of results\"\"\"\n        if self.results is None or len(self.results) == 0:\n            print(\"No results to visualize\")\n            return\n        \n        # Set up the plot\n        fig = plt.figure(figsize=(20, 16))\n        \n        # 1. Performance comparison (Pearson correlation)\n        ax1 = plt.subplot(3, 3, 1)\n        top_methods = self.results.head(20)\n        colors = ['green' if x == 'feature_selection' else 'blue' if x == 'unsupervised' else 'red' \n                  for x in top_methods['method_type']]\n        ax1.barh(range(len(top_methods)), top_methods['pearson'], color=colors)\n        ax1.set_yticks(range(len(top_methods)))\n        ax1.set_yticklabels(top_methods['method'], fontsize=8)\n        ax1.set_xlabel('Pearson Correlation')\n        ax1.set_title('Top 20 Methods by Pearson Correlation')\n        ax1.invert_yaxis()\n        \n        # 2. R² comparison\n        ax2 = plt.subplot(3, 3, 2)\n        ax2.barh(range(len(top_methods)), top_methods['r2'], color=colors)\n        ax2.set_yticks(range(len(top_methods)))\n        ax2.set_yticklabels(top_methods['method'], fontsize=8)\n        ax2.set_xlabel('R² Score')\n        ax2.set_title('Top 20 Methods by R²')\n        ax2.invert_yaxis()\n        \n        # 3. RMSE comparison (lower is better)\n        ax3 = plt.subplot(3, 3, 3)\n        top_methods_rmse = self.results.sort_values('rmse').head(20)\n        colors_rmse = ['green' if x == 'feature_selection' else 'blue' if x == 'unsupervised' else 'red' \n                       for x in top_methods_rmse['method_type']]\n        ax3.barh(range(len(top_methods_rmse)), top_methods_rmse['rmse'], color=colors_rmse)\n        ax3.set_yticks(range(len(top_methods_rmse)))\n        ax3.set_yticklabels(top_methods_rmse['method'], fontsize=8)\n        ax3.set_xlabel('RMSE')\n        ax3.set_title('Top 20 Methods by RMSE (Lower is Better)')\n        ax3.invert_yaxis()\n        \n        # 4. Method type distribution\n        ax4 = plt.subplot(3, 3, 4)\n        method_counts = self.results['method_type'].value_counts()\n        ax4.pie(method_counts.values, labels=method_counts.index, autopct='%1.1f%%')\n        ax4.set_title('Distribution of Method Types')\n        \n        # 5. Performance vs Features scatter\n        ax5 = plt.subplot(3, 3, 5)\n        for method_type in self.results['method_type'].unique():\n            mask = self.results['method_type'] == method_type\n            ax5.scatter(self.results[mask]['n_features'], \n                       self.results[mask]['pearson'],\n                       label=method_type, alpha=0.6, s=50)\n        ax5.set_xlabel('Number of Features')\n        ax5.set_ylabel('Pearson Correlation')\n        ax5.set_title('Performance vs Number of Features')\n        ax5.legend()\n        ax5.grid(True, alpha=0.3)\n        \n        # 6. Computation time comparison\n        ax6 = plt.subplot(3, 3, 6)\n        top_methods_time = self.results.sort_values('pearson', ascending=False).head(20)\n        total_time = top_methods_time['reduction_time'] + top_methods_time['train_time']\n        ax6.barh(range(len(top_methods_time)), total_time)\n        ax6.set_yticks(range(len(top_methods_time)))\n        ax6.set_yticklabels(top_methods_time['method'], fontsize=8)\n        ax6.set_xlabel('Total Time (seconds)')\n        ax6.set_title('Computation Time for Top 20 Methods')\n        ax6.invert_yaxis()\n        \n        # 7. Performance heatmap by method type\n        ax7 = plt.subplot(3, 3, 7)\n        pivot_data = self.results.pivot_table(\n            values=['pearson', 'r2', 'rmse'],\n            index='method_type',\n            aggfunc='mean'\n        )\n        sns.heatmap(pivot_data.T, annot=True, fmt='.3f', cmap='coolwarm', ax=ax7)\n        ax7.set_title('Average Performance by Method Type')\n        \n        # 8. Top methods summary\n        ax8 = plt.subplot(3, 3, 8)\n        ax8.axis('off')\n        top_5 = self.results.head(5)\n        summary_text = \"TOP 5 METHODS:\\n\" + \"=\"*40 + \"\\n\"\n        for idx, row in top_5.iterrows():\n            summary_text += f\"{row['method']}\\n\"\n            summary_text += f\"  Pearson: {row['pearson']:.4f}\\n\"\n            summary_text += f\"  R²: {row['r2']:.4f}\\n\"\n            summary_text += f\"  RMSE: {row['rmse']:.4f}\\n\"\n            summary_text += f\"  Features: {row['n_features']}\\n\"\n            summary_text += \"-\"*40 + \"\\n\"\n        ax8.text(0.1, 0.9, summary_text, transform=ax8.transAxes, \n                fontsize=10, verticalalignment='top', family='monospace')\n        \n        # 9. Method type performance comparison\n        ax9 = plt.subplot(3, 3, 9)\n        self.results.boxplot(column='pearson', by='method_type', ax=ax9)\n        ax9.set_title('Performance Distribution by Method Type')\n        ax9.set_xlabel('Method Type')\n        ax9.set_ylabel('Pearson Correlation')\n        \n        plt.suptitle('Comprehensive Dimensionality Reduction Analysis', fontsize=16)\n        plt.tight_layout()\n        plt.savefig('dimensionality_reduction_analysis.png', dpi=300, bbox_inches='tight')\n        plt.close()\n        \n        # Create additional detailed plots\n        self.create_detailed_comparisons()\n    \n    def create_detailed_comparisons(self):\n        \"\"\"Create additional detailed comparison plots\"\"\"\n        # Plot 1: Feature selection methods comparison\n        fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n        \n        # Feature selection only\n        fs_results = self.results[self.results['method_type'] == 'feature_selection'].copy()\n        \n        if len(fs_results) > 0:\n            ax1 = axes[0, 0]\n            fs_results = fs_results.sort_values('pearson', ascending=False)\n            ax1.barh(range(len(fs_results)), fs_results['pearson'])\n            ax1.set_yticks(range(len(fs_results)))\n            ax1.set_yticklabels(fs_results['method'], fontsize=10)\n            ax1.set_xlabel('Pearson Correlation')\n            ax1.set_title('Feature Selection Methods Comparison')\n            ax1.invert_yaxis()\n        \n        # Unsupervised methods\n        unsup_results = self.results[self.results['method_type'] == 'unsupervised'].copy()\n        \n        if len(unsup_results) > 0:\n            ax2 = axes[0, 1]\n            unsup_results = unsup_results.sort_values('pearson', ascending=False)\n            ax2.barh(range(len(unsup_results)), unsup_results['pearson'])\n            ax2.set_yticks(range(len(unsup_results)))\n            ax2.set_yticklabels(unsup_results['method'], fontsize=10)\n            ax2.set_xlabel('Pearson Correlation')\n            ax2.set_title('Unsupervised Methods Comparison')\n            ax2.invert_yaxis()\n        \n        # Performance vs Reduction Time\n        ax3 = axes[1, 0]\n        ax3.scatter(self.results['reduction_time'], self.results['pearson'], alpha=0.6)\n        for idx, row in self.results.head(5).iterrows():\n            ax3.annotate(row['method'], (row['reduction_time'], row['pearson']), fontsize=8)\n        ax3.set_xlabel('Reduction Time (seconds)')\n        ax3.set_ylabel('Pearson Correlation')\n        ax3.set_title('Performance vs Computation Time')\n        ax3.grid(True, alpha=0.3)\n        \n        # Summary statistics\n        ax4 = axes[1, 1]\n        ax4.axis('off')\n        \n        summary_stats = f\"\"\"\nSUMMARY STATISTICS:\n{'='*40}\nTotal methods tested: {len(self.results)}\nBest method: {self.best_method}\nBest Pearson: {self.results.iloc[0]['pearson']:.4f}\nBest R²: {self.results['r2'].max():.4f}\nBest RMSE: {self.results['rmse'].min():.4f}\n\nAverage by Type:\n{'-'*40}\n\"\"\"\n        for method_type in self.results['method_type'].unique():\n            type_results = self.results[self.results['method_type'] == method_type]\n            summary_stats += f\"{method_type}:\\n\"\n            summary_stats += f\"  Avg Pearson: {type_results['pearson'].mean():.4f}\\n\"\n            summary_stats += f\"  Avg Features: {type_results['n_features'].mean():.1f}\\n\"\n        \n        ax4.text(0.1, 0.9, summary_stats, transform=ax4.transAxes,\n                fontsize=11, verticalalignment='top', family='monospace')\n        \n        plt.tight_layout()\n        plt.savefig('detailed_method_comparison.png', dpi=200, bbox_inches='tight')\n        plt.close()\n    \n    def export_results(self):\n        \"\"\"Export results to CSV files\"\"\"\n        # Save main results\n        self.results.to_csv('dimensionality_reduction_results.csv', index=False)\n        \n        # Save top methods\n        self.results.head(10).to_csv('top_10_methods.csv', index=False)\n        \n        # Save method type summary\n        summary = self.results.groupby('method_type').agg({\n            'pearson': ['mean', 'std', 'max'],\n            'r2': ['mean', 'std', 'max'],\n            'rmse': ['mean', 'std', 'min'],\n            'n_features': ['mean', 'std'],\n            'reduction_time': ['mean', 'std']\n        }).round(4)\n        summary.to_csv('method_type_summary.csv')\n        \n        print(\"\\nResults saved:\")\n        print(\"  - dimensionality_reduction_results.csv\")\n        print(\"  - top_10_methods.csv\")\n        print(\"  - method_type_summary.csv\")\n\ndef main():\n    \"\"\"Main execution\"\"\"\n    print(\"=\"*60)\n    print(\"FINANCIAL DATA DIMENSIONALITY REDUCTION EXPERIMENTS\")\n    print(f\"Started at: {datetime.now()}\")\n    print(\"=\"*60)\n    \n    # Load data\n    print(\"\\nLoading data...\")\n    train_path = os.path.join(DATA_PATH, 'train.parquet')\n    df = pd.read_parquet(train_path)\n    print(f\"Total dataset size: {len(df):,} rows\")\n    \n    # Initialize reducer\n    reducer = FinancialDimensionalityReducer(\n        n_components=50,  # Target dimensions\n        sample_size=10000,  # Use 10k samples as requested\n        n_jobs=4\n    )\n    \n    # Optimize memory\n    print(\"\\nOptimizing memory...\")\n    df = reducer.optimize_dtypes(df)\n    \n    # Get feature columns\n    feature_cols = [col for col in df.columns if col not in ['timestamp', 'label']]\n    print(f\"Number of features: {len(feature_cols)}\")\n    \n    # Run experiments\n    start_time = time.time()\n    results = reducer.run_experiments(df, feature_cols)\n    total_time = time.time() - start_time\n    \n    print(f\"\\n{'='*60}\")\n    print(\"EXPERIMENT RESULTS\")\n    print('='*60)\n    print(f\"Total processing time: {total_time/60:.1f} minutes\")\n    print(f\"Methods tested: {len(results)}\")\n    print(f\"Best method: {reducer.best_method}\")\n    \n    # Show top 10 results\n    print(\"\\nTop 10 Methods:\")\n    print(\"-\"*80)\n    print(f\"{'Method':<30} {'Type':<15} {'Pearson':<10} {'R²':<10} {'RMSE':<10} {'Features':<10}\")\n    print(\"-\"*80)\n    \n    for _, row in results.head(10).iterrows():\n        print(f\"{row['method']:<30} {row['method_type']:<15} \"\n              f\"{row['pearson']:<10.4f} {row['r2']:<10.4f} \"\n              f\"{row['rmse']:<10.4f} {int(row['n_features']):<10}\")\n    \n    # Create visualizations\n    print(\"\\nCreating visualizations...\")\n    reducer.create_comprehensive_visualization()\n    \n    # Export results\n    reducer.export_results()\n    \n    # Final recommendations\n    print(\"\\n\" + \"=\"*60)\n    print(\"RECOMMENDATIONS\")\n    print(\"=\"*60)\n    \n    top_3 = results.head(3)\n    for idx, (_, row) in enumerate(top_3.iterrows(), 1):\n        print(f\"\\n{idx}. {row['method']} ({row['method_type']})\")\n        print(f\"   - Pearson: {row['pearson']:.4f}\")\n        print(f\"   - Features: {int(row['n_features'])}\")\n        print(f\"   - Reduction time: {row['reduction_time']:.2f}s\")\n    \n    print(f\"\\nCompleted at: {datetime.now()}\")\n    print(\"=\"*60)\n    \n    return reducer, results\n\nif __name__ == \"__main__\":\n    reducer, results = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}