{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install gputil","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Complete Enhanced Crypto Market Prediction Model with Anti-Overfitting Strategies\n# Fixed version with all error corrections\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport gc\nimport os\nfrom typing import List, Dict, Tuple, Optional\nfrom sklearn.preprocessing import KBinsDiscretizer, StandardScaler\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr, kurtosis, rankdata\nfrom sklearn.feature_selection import mutual_info_regression\nimport warnings\nfrom scipy.special import expit\nimport random\nfrom sklearn.metrics import mean_squared_error\nwarnings.filterwarnings('ignore')\n\n# ===== Enhanced Configuration =====\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Bucketing parameters\n    MIN_BUCKETS = 3\n    MAX_BUCKETS = 15\n    DEFAULT_BUCKETS = 10\n    \n    # GPU Configuration\n    USE_GPU = torch.cuda.is_available()\n    N_GPUS = torch.cuda.device_count() if USE_GPU else 0\n    \n    # Memory management\n    CHUNK_SIZE = 100000\n    MAX_FEATURES_PER_BATCH = 200\n    \n    # Feature selection\n    TOP_FEATURES_TO_SELECT = 85\n    \n    # Enhanced Training parameters\n    USE_BUCKETED_FEATURES = True\n    N_FOLDS = 5\n    N_SEEDS = 3\n    RANDOM_STATE = 42\n    LABEL_COLUMN = \"label\"\n    \n    # Anti-overfitting parameters\n    NOISE_LEVEL = 0.01\n    DROPOUT_RATE = 0.1\n    MIN_SAMPLES_LEAF_RATIO = 0.02\n    EARLY_STOPPING_ROUNDS = 100\n    USE_PURGED_SPLIT = True\n    PURGE_LENGTH = 1000\n    \n    # Ensemble parameters\n    USE_RANK_BLEND = True\n    USE_EXTREMES_CLIPPING = True\n    CLIP_PERCENTILE = 0.1\n\nprint(f\"GPU Available: {Config.USE_GPU}\")\nprint(f\"Number of GPUs: {Config.N_GPUS}\")\n\n# ===== Step 0: Remove Uninformative Columns =====\ndef remove_uninformative_columns(train_df: pd.DataFrame, test_df: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame, List[str]]:\n    \"\"\"Remove columns with no information\"\"\"\n    print(\"=\" * 80)\n    print(\"STEP 0: Removing uninformative columns\")\n    print(\"=\" * 80)\n    \n    initial_cols = train_df.columns.tolist()\n    if Config.LABEL_COLUMN in initial_cols:\n        initial_cols.remove(Config.LABEL_COLUMN)\n    \n    removed_columns = []\n    kept_columns = []\n    \n    for col in initial_cols:\n        if col not in test_df.columns:\n            removed_columns.append((col, \"missing_in_test\"))\n            continue\n        \n        train_unique = train_df[col].nunique()\n        if train_unique <= 1:\n            removed_columns.append((col, \"constant_in_train\"))\n            continue\n        \n        test_unique = test_df[col].nunique()\n        if test_unique <= 1:\n            removed_columns.append((col, \"constant_in_test\"))\n            continue\n        \n        if train_df[col].isna().all() or test_df[col].isna().all():\n            removed_columns.append((col, \"all_nan\"))\n            continue\n        \n        kept_columns.append(col)\n    \n    print(f\"Initial columns: {len(initial_cols)}\")\n    print(f\"Removed columns: {len(removed_columns)}\")\n    print(f\"Kept columns: {len(kept_columns)}\")\n    \n    train_df_clean = train_df[kept_columns + ([Config.LABEL_COLUMN] if Config.LABEL_COLUMN in train_df else [])]\n    test_df_clean = test_df[kept_columns]\n    \n    return train_df_clean, test_df_clean, kept_columns\n\n# ===== Enhanced Intelligent Bucketing with Regularization =====\nclass EnhancedIntelligentBucketer:\n    \"\"\"Enhanced bucketer with noise injection and regularization\"\"\"\n    \n    def __init__(self, min_buckets=3, max_buckets=15, noise_level=0.0):\n        self.min_buckets = min_buckets\n        self.max_buckets = max_buckets\n        self.noise_level = noise_level\n        self.bucket_info = {}\n        self.discretizers = {}\n        self.feature_stats = {}\n        \n    def _determine_optimal_buckets(self, feature_values: np.ndarray) -> int:\n        \"\"\"Determine optimal number of buckets with conservative approach\"\"\"\n        clean_values = feature_values[~np.isnan(feature_values)]\n        if len(clean_values) == 0:\n            return self.min_buckets\n        \n        n_unique = len(np.unique(clean_values))\n        \n        if n_unique <= self.max_buckets:\n            return min(n_unique, self.max_buckets)\n        \n        n_samples = len(clean_values)\n        \n        # More conservative bucket calculation\n        sturges = int(np.ceil(np.log2(n_samples) + 1))\n        \n        # Freedman-Diaconis with error handling\n        q75, q25 = np.percentile(clean_values, [75, 25])\n        iqr = q75 - q25\n        if iqr > 0 and n_samples > 0:\n            fd_bins = int(2 * iqr / (n_samples ** (1/3)))\n            fd_bins = max(self.min_buckets, min(fd_bins, self.max_buckets))\n        else:\n            fd_bins = self.min_buckets\n        \n        # Take minimum for more conservative bucketing\n        n_buckets = min(sturges, fd_bins, self.max_buckets)\n        n_buckets = max(self.min_buckets, n_buckets)\n        \n        return n_buckets\n    \n    def _handle_infinities(self, values: np.ndarray) -> np.ndarray:\n        \"\"\"Handle infinite values\"\"\"\n        values = values.copy()\n        \n        finite_mask = np.isfinite(values)\n        if not np.any(finite_mask):\n            return np.zeros_like(values)\n        \n        finite_values = values[finite_mask]\n        \n        if np.any(values == np.inf):\n            max_finite = np.percentile(finite_values, 99)\n            values[values == np.inf] = max_finite\n        \n        if np.any(values == -np.inf):\n            min_finite = np.percentile(finite_values, 1)\n            values[values == -np.inf] = min_finite\n        \n        return values\n    \n    def _add_noise(self, values: np.ndarray) -> np.ndarray:\n        \"\"\"Add small Gaussian noise for regularization\"\"\"\n        if self.noise_level > 0:\n            std_val = np.std(values[np.isfinite(values)])\n            if std_val > 0:\n                noise = np.random.normal(0, self.noise_level * std_val, values.shape)\n                return values + noise\n        return values\n    \n    def fit_transform(self, df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:\n        \"\"\"Fit and transform features into buckets with regularization\"\"\"\n        if columns is None:\n            columns = df.columns.tolist()\n            if Config.LABEL_COLUMN in columns:\n                columns.remove(Config.LABEL_COLUMN)\n        \n        print(f\"\\nEnhanced bucketing {len(columns)} features...\")\n        df_bucketed = df.copy()\n        \n        for i, col in enumerate(columns):\n            if i % 100 == 0 and i > 0:\n                print(f\"  Progress: {i}/{len(columns)}\")\n            \n            values = self._handle_infinities(df[col].values)\n            \n            # Store feature statistics for later use\n            finite_vals = values[np.isfinite(values)]\n            if len(finite_vals) > 0:\n                self.feature_stats[col] = {\n                    'mean': np.mean(finite_vals),\n                    'std': np.std(finite_vals),\n                    'median': np.median(finite_vals)\n                }\n            else:\n                self.feature_stats[col] = {\n                    'mean': 0,\n                    'std': 1,\n                    'median': 0\n                }\n            \n            if np.all(np.isnan(values)) or len(finite_vals) == 0:\n                df_bucketed[col] = 0\n                continue\n            \n            # Add noise during training\n            values_noisy = self._add_noise(values)\n            \n            n_buckets = self._determine_optimal_buckets(values)\n            \n            try:\n                discretizer = KBinsDiscretizer(\n                    n_bins=n_buckets,\n                    encode='ordinal',\n                    strategy='quantile'\n                )\n                \n                median_val = self.feature_stats[col]['median']\n                values_filled = np.nan_to_num(values_noisy.reshape(-1, 1), nan=median_val)\n                \n                bucketed = discretizer.fit_transform(values_filled).flatten()\n                \n                self.discretizers[col] = discretizer\n                self.bucket_info[col] = {\n                    'n_buckets': n_buckets,\n                    'unique_buckets': len(np.unique(bucketed))\n                }\n                \n                df_bucketed[col] = bucketed.astype(np.int8)\n                \n            except Exception as e:\n                df_bucketed[col] = 0\n        \n        print(f\"  Progress: {len(columns)}/{len(columns)}\")\n        return df_bucketed\n    \n    def transform(self, df: pd.DataFrame, columns: Optional[List[str]] = None) -> pd.DataFrame:\n        \"\"\"Transform new data (no noise injection)\"\"\"\n        if columns is None:\n            columns = list(self.discretizers.keys())\n        \n        df_bucketed = df.copy()\n        \n        for col in columns:\n            if col in self.discretizers and col in self.feature_stats:\n                values = self._handle_infinities(df[col].values)\n                \n                try:\n                    median_val = self.feature_stats[col]['median']\n                    values_filled = np.nan_to_num(values.reshape(-1, 1), nan=median_val)\n                    \n                    bucketed = self.discretizers[col].transform(values_filled).flatten()\n                    df_bucketed[col] = bucketed.astype(np.int8)\n                except:\n                    df_bucketed[col] = 0\n            else:\n                df_bucketed[col] = 0\n        \n        return df_bucketed\n\n# ===== Enhanced Feature Selection with Stability =====\ndef evaluate_features_with_stability(X_train: np.ndarray, y_train: np.ndarray, \n                                   feature_names: List[str], n_runs: int = 3) -> Dict[str, float]:\n    \"\"\"Evaluate features multiple times for stability\"\"\"\n    n_features = X_train.shape[1]\n    chunk_size = Config.MAX_FEATURES_PER_BATCH\n    n_chunks = (n_features + chunk_size - 1) // chunk_size\n    \n    print(f\"Evaluating {n_features} features with {n_runs} stability runs...\")\n    \n    all_importances_runs = []\n    \n    # Sample size for faster evaluation\n    sample_size = min(100000, len(X_train))\n    \n    for run in range(n_runs):\n        print(f\"\\nStability run {run + 1}/{n_runs}\")\n        all_importances = {}\n        \n        # Different random sample each run\n        if len(X_train) > sample_size:\n            sample_idx = np.random.choice(len(X_train), sample_size, replace=False)\n            X_sample = X_train[sample_idx]\n            y_sample = y_train[sample_idx]\n        else:\n            X_sample = X_train\n            y_sample = y_train\n        \n        for chunk_idx in range(n_chunks):\n            start_idx = chunk_idx * chunk_size\n            end_idx = min((chunk_idx + 1) * chunk_size, n_features)\n            \n            chunk_features = feature_names[start_idx:end_idx]\n            X_chunk = X_sample[:, start_idx:end_idx]\n            \n            if chunk_idx % 5 == 0:\n                print(f\"  Chunk {chunk_idx + 1}/{n_chunks}\")\n            \n            # More conservative XGBoost parameters for feature selection\n            xgb_params = {\n                'tree_method': 'gpu_hist' if Config.USE_GPU else 'hist',\n                'gpu_id': 0 if Config.USE_GPU else None,\n                'max_depth': 5,\n                'n_estimators': 100,\n                'learning_rate': 0.05,\n                'subsample': 0.7,\n                'colsample_bytree': 0.7,\n                'min_child_weight': 10,\n                'gamma': 0.1,\n                'reg_alpha': 1.0,\n                'reg_lambda': 1.0,\n                'random_state': 42 + run,\n                'verbosity': 0\n            }\n            \n            if not Config.USE_GPU:\n                xgb_params['n_jobs'] = -1\n                del xgb_params['gpu_id']\n            \n            try:\n                model = XGBRegressor(**xgb_params)\n                model.fit(X_chunk, y_sample)\n                \n                importances = model.feature_importances_\n                \n                for feat_name, imp in zip(chunk_features, importances):\n                    all_importances[feat_name] = imp\n                \n                del model\n                gc.collect()\n            except Exception as e:\n                print(f\"    Warning: Error in chunk {chunk_idx}: {str(e)}\")\n                for feat_name in chunk_features:\n                    all_importances[feat_name] = 0.0\n        \n        all_importances_runs.append(all_importances)\n    \n    # Average importances across runs\n    final_importances = {}\n    for feat in feature_names:\n        scores = [run_imp.get(feat, 0.0) for run_imp in all_importances_runs]\n        # Use mean but penalize high variance features\n        mean_score = np.mean(scores)\n        std_score = np.std(scores)\n        if mean_score > 0:\n            final_importances[feat] = mean_score * (1 - 0.1 * std_score / (mean_score + 1e-8))\n        else:\n            final_importances[feat] = 0.0\n    \n    return final_importances\n\n# ===== Purged Time Series Split =====\nclass PurgedKFold:\n    \"\"\"Time series cross-validation with purging\"\"\"\n    \n    def __init__(self, n_splits=3, purge_length=0):\n        self.n_splits = n_splits\n        self.purge_length = purge_length\n    \n    def split(self, X, y=None, groups=None):\n        n_samples = len(X)\n        indices = np.arange(n_samples)\n        \n        fold_size = n_samples // self.n_splits\n        \n        for i in range(self.n_splits):\n            val_start = i * fold_size\n            val_end = (i + 1) * fold_size if i < self.n_splits - 1 else n_samples\n            \n            # Training data: everything before validation (with purge gap)\n            train_end = val_start - self.purge_length\n            train_indices = indices[:train_end] if train_end > 0 else np.array([], dtype=int)\n            \n            val_indices = indices[val_start:val_end]\n            \n            if len(train_indices) > 0:\n                yield train_indices, val_indices\n\n# ===== Fixed Time Decay Weights Function =====\ndef create_adaptive_time_weights(n: int, y: np.ndarray, decay: float = 0.9) -> np.ndarray:\n    \"\"\"Create adaptive time decay weights based on target volatility - FIXED VERSION\"\"\"\n    positions = np.arange(n)\n    \n    # Handle edge case where n = 1\n    if n <= 1:\n        return np.ones(n)\n    \n    normalized = positions / (n - 1)\n    \n    # Base exponential decay\n    base_weights = decay ** (1.0 - normalized)\n    \n    # Adaptive component based on local volatility\n    window_size = max(10, min(100, n // 20))\n    \n    try:\n        # Calculate rolling volatility with error handling\n        volatility = pd.Series(y).rolling(window_size, min_periods=1).std()\n        \n        # Fill NaN values with median volatility\n        volatility = volatility.fillna(volatility.median())\n        \n        # If all volatilities are NaN or zero, use base weights\n        if volatility.isna().all() or volatility.max() == 0:\n            weights = base_weights\n        else:\n            # Normalize volatility\n            volatility_normalized = volatility / (volatility.max() + 1e-8)\n            volatility_normalized = volatility_normalized.fillna(0.5)  # Default to 0.5 if still NaN\n            \n            # Higher weight for high volatility periods\n            adaptive_weights = base_weights * (1 + 0.5 * volatility_normalized.values)\n            weights = adaptive_weights\n    except Exception as e:\n        # Fallback to base weights if any error occurs\n        weights = base_weights\n    \n    # Ensure all weights are positive\n    weights = np.maximum(weights, 1e-8)\n    \n    # Normalize to sum to n\n    weight_sum = weights.sum()\n    if weight_sum > 0:\n        return weights * n / weight_sum\n    else:\n        # Fallback to uniform weights if something goes wrong\n        return np.ones(n)\n\n# ===== Feature Dropout =====\ndef apply_feature_dropout(X: np.ndarray, dropout_rate: float = 0.1) -> np.ndarray:\n    \"\"\"Apply feature dropout for regularization\"\"\"\n    if dropout_rate > 0 and X.shape[1] > 1:\n        n_features_to_keep = max(1, int(X.shape[1] * (1 - dropout_rate)))\n        selected_features = np.random.choice(X.shape[1], n_features_to_keep, replace=False)\n        X_dropout = np.zeros_like(X)\n        X_dropout[:, selected_features] = X[:, selected_features]\n        return X_dropout\n    return X\n\n# ===== Enhanced Model Training =====\ndef train_with_enhanced_cv(train_df, test_df, features):\n    \"\"\"Enhanced training with multiple anti-overfitting strategies\"\"\"\n    n_samples = len(train_df)\n    \n    # Model configurations with different regularization levels\n    model_configs = [\n        {\n            \"name\": \"conservative\",\n            \"params\": {\n                'tree_method': 'gpu_hist' if Config.USE_GPU else 'hist',\n                'n_jobs': -1,\n                'colsample_bytree': 0.3,\n                'colsample_bynode': 0.3,\n                'gamma': 2.0,\n                'learning_rate': 0.01,\n                'max_depth': 5,\n                'max_leaves': 20,\n                'n_estimators': 1000,\n                'reg_alpha': 50,\n                'reg_lambda': 100,\n                'subsample': 0.5,\n                'min_child_weight': 50,\n                'random_state': Config.RANDOM_STATE\n            }\n        },\n        {\n            \"name\": \"moderate\",\n            \"params\": {\n                'tree_method': 'gpu_hist' if Config.USE_GPU else 'hist',\n                'n_jobs': -1,\n                'colsample_bytree': 0.4,\n                'colsample_bynode': 0.3,\n                'gamma': 1.5,\n                'learning_rate': 0.015,\n                'max_depth': 6,\n                'max_leaves': 30,\n                'n_estimators': 800,\n                'reg_alpha': 30,\n                'reg_lambda': 80,\n                'subsample': 0.6,\n                'min_child_weight': 30,\n                'random_state': Config.RANDOM_STATE\n            }\n        },\n        {\n            \"name\": \"aggressive\",\n            \"params\": {\n                'tree_method': 'gpu_hist' if Config.USE_GPU else 'hist',\n                'n_jobs': -1,\n                'colsample_bytree': 0.45,\n                'colsample_bynode': 0.35,\n                'gamma': 1.0,\n                'learning_rate': 0.02,\n                'max_depth': 7,\n                'max_leaves': 40,\n                'n_estimators': 600,\n                'reg_alpha': 20,\n                'reg_lambda': 60,\n                'subsample': 0.7,\n                'min_child_weight': 20,\n                'random_state': Config.RANDOM_STATE\n            }\n        }\n    ]\n    \n    # Time slices for ensemble\n    time_slices = [\n        {\"name\": \"recent_30pct\", \"start_pct\": 0.7},\n        {\"name\": \"recent_50pct\", \"start_pct\": 0.5},\n        {\"name\": \"recent_70pct\", \"start_pct\": 0.3},\n        {\"name\": \"full_data\", \"start_pct\": 0.0}\n    ]\n    \n    all_oof_preds = {}\n    all_test_preds = {}\n    \n    # Multiple seeds for stability\n    for seed_idx in range(Config.N_SEEDS):\n        print(f\"\\n{'='*60}\")\n        print(f\"SEED {seed_idx + 1}/{Config.N_SEEDS}\")\n        print(f\"{'='*60}\")\n        \n        np.random.seed(Config.RANDOM_STATE + seed_idx)\n        \n        # Use purged cross-validation\n        if Config.USE_PURGED_SPLIT:\n            cv = PurgedKFold(n_splits=Config.N_FOLDS, purge_length=Config.PURGE_LENGTH)\n        else:\n            cv = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n        \n        for config in model_configs:\n            config_name = config[\"name\"]\n            params = config[\"params\"].copy()\n            params['random_state'] = Config.RANDOM_STATE + seed_idx\n            \n            if Config.USE_GPU:\n                params['gpu_id'] = 0\n            \n            print(f\"\\nTraining {config_name} model...\")\n            \n            for time_slice in time_slices:\n                slice_name = f\"{config_name}_{time_slice['name']}_seed{seed_idx}\"\n                start_idx = int(n_samples * time_slice['start_pct'])\n                \n                # Get slice data\n                slice_df = train_df.iloc[start_idx:].reset_index(drop=True)\n                slice_n = len(slice_df)\n                \n                oof_preds = np.zeros(n_samples)\n                test_preds = np.zeros(len(test_df))\n                \n                print(f\"  Time slice: {time_slice['name']} (n={slice_n})\")\n                \n                fold_scores = []\n                fold_count = 0\n                \n                for fold, (train_idx, valid_idx) in enumerate(cv.split(slice_df), start=1):\n                    # Adjust indices for original dataframe\n                    train_idx_orig = train_idx + start_idx\n                    valid_idx_orig = valid_idx + start_idx\n                    \n                    # Skip if no training data or validation data\n                    if len(train_idx) == 0 or len(valid_idx) == 0:\n                        continue\n                    \n                    # Get data\n                    X_train = train_df.iloc[train_idx_orig][features].values\n                    y_train = train_df.iloc[train_idx_orig][Config.LABEL_COLUMN].values\n                    X_valid = train_df.iloc[valid_idx_orig][features].values\n                    y_valid = train_df.iloc[valid_idx_orig][Config.LABEL_COLUMN].values\n                    \n                    # Apply feature dropout to training data\n                    X_train_dropout = apply_feature_dropout(X_train, Config.DROPOUT_RATE)\n                    \n                    # Create adaptive weights with error handling\n                    try:\n                        weights = create_adaptive_time_weights(len(y_train), y_train)\n                    except Exception as e:\n                        print(f\"    Warning: Error creating weights, using uniform weights: {str(e)}\")\n                        weights = np.ones(len(y_train))\n                    \n                    # Train model\n                    try:\n                        model = XGBRegressor(**params)\n                        model.fit(\n                            X_train_dropout, y_train,\n                            sample_weight=weights,\n                            eval_set=[(X_valid, y_valid)],\n                            early_stopping_rounds=Config.EARLY_STOPPING_ROUNDS,\n                            verbose=False\n                        )\n                        \n                        # Predictions\n                        valid_preds = model.predict(X_valid)\n                        oof_preds[valid_idx_orig] = valid_preds\n                        \n                        # Test predictions (average across folds)\n                        fold_count += 1\n                        test_preds += model.predict(test_df[features].values) / Config.N_FOLDS\n                        \n                        # Calculate fold score\n                        fold_score = pearsonr(y_valid, valid_preds)[0]\n                        fold_scores.append(fold_score)\n                        \n                        if fold <= 2:  # Print first 2 folds\n                            print(f\"    Fold {fold}: {fold_score:.4f}\")\n                        \n                        del model\n                        gc.collect()\n                        \n                    except Exception as e:\n                        print(f\"    Warning: Error in fold {fold}: {str(e)}\")\n                        continue\n                \n                # Adjust test predictions if not all folds were used\n                if fold_count > 0 and fold_count != Config.N_FOLDS:\n                    test_preds = test_preds * Config.N_FOLDS / fold_count\n                \n                # Store predictions\n                all_oof_preds[slice_name] = oof_preds\n                all_test_preds[slice_name] = test_preds\n                \n                # Print average score\n                if fold_scores:\n                    avg_score = np.mean(fold_scores)\n                    print(f\"    Average: {avg_score:.4f}\")\n    \n    # Create ensemble predictions\n    print(\"\\n\" + \"=\"*60)\n    print(\"CREATING ENSEMBLE\")\n    print(\"=\"*60)\n    \n    # Calculate OOF scores for weighting\n    oof_scores = {}\n    for name, preds in all_oof_preds.items():\n        # Only score where we have predictions\n        mask = preds != 0\n        if mask.sum() > 0:\n            try:\n                score = pearsonr(train_df.loc[mask, Config.LABEL_COLUMN], preds[mask])[0]\n                if np.isnan(score):\n                    score = 0.0\n                oof_scores[name] = score\n                print(f\"{name}: {score:.4f}\")\n            except:\n                oof_scores[name] = 0.0\n    \n    # Rank blend if enabled\n    if Config.USE_RANK_BLEND and len(oof_scores) > 0:\n        print(\"\\nUsing rank-based blending...\")\n        \n        # Convert to ranks for OOF\n        oof_ranks = {}\n        for name, preds in all_oof_preds.items():\n            mask = preds != 0\n            ranks = np.zeros_like(preds)\n            if mask.sum() > 0:\n                ranks[mask] = rankdata(preds[mask]) / mask.sum()\n            oof_ranks[name] = ranks\n        \n        # Convert to ranks for test\n        test_ranks = {}\n        for name, preds in all_test_preds.items():\n            test_ranks[name] = rankdata(preds) / len(preds)\n        \n        # Weighted average of ranks\n        weights = np.array([max(0, oof_scores.get(name, 0)) for name in all_test_preds.keys()])\n        if weights.sum() > 0:\n            weights = weights / weights.sum()\n        else:\n            weights = np.ones(len(all_test_preds)) / len(all_test_preds)\n        \n        ensemble_oof_ranks = np.zeros(n_samples)\n        ensemble_test_ranks = np.zeros(len(test_df))\n        \n        for i, name in enumerate(all_test_preds.keys()):\n            ensemble_oof_ranks += weights[i] * oof_ranks.get(name, np.zeros(n_samples))\n            ensemble_test_ranks += weights[i] * test_ranks[name]\n        \n        # Convert ranks back to predictions using quantile mapping\n        train_labels_sorted = np.sort(train_df[Config.LABEL_COLUMN].values)\n        ensemble_oof = np.interp(ensemble_oof_ranks, np.linspace(0, 1, len(train_labels_sorted)), train_labels_sorted)\n        ensemble_test = np.interp(ensemble_test_ranks, np.linspace(0, 1, len(train_labels_sorted)), train_labels_sorted)\n        \n    else:\n        # Simple weighted average\n        weights = np.array([max(0, oof_scores.get(name, 0)) for name in all_test_preds.keys()])\n        if weights.sum() > 0:\n            weights = weights / weights.sum()\n        else:\n            weights = np.ones(len(all_test_preds)) / len(all_test_preds)\n        \n        ensemble_oof = np.zeros(n_samples)\n        ensemble_test = np.zeros(len(test_df))\n        \n        for i, name in enumerate(all_test_preds.keys()):\n            ensemble_oof += weights[i] * all_oof_preds.get(name, np.zeros(n_samples))\n            ensemble_test += weights[i] * all_test_preds[name]\n    \n    # Clip extreme predictions if enabled\n    if Config.USE_EXTREMES_CLIPPING:\n        print(\"\\nClipping extreme predictions...\")\n        lower = np.percentile(train_df[Config.LABEL_COLUMN], Config.CLIP_PERCENTILE)\n        upper = np.percentile(train_df[Config.LABEL_COLUMN], 100 - Config.CLIP_PERCENTILE)\n        ensemble_test = np.clip(ensemble_test, lower, upper)\n    \n    # Final ensemble score\n    try:\n        ensemble_score = pearsonr(train_df[Config.LABEL_COLUMN], ensemble_oof)[0]\n        if np.isnan(ensemble_score):\n            ensemble_score = 0.0\n    except:\n        ensemble_score = 0.0\n    \n    print(f\"\\nFinal Ensemble Score: {ensemble_score:.4f}\")\n    \n    return {\n        'ensemble_test': ensemble_test,\n        'ensemble_score': ensemble_score,\n        'all_test_preds': all_test_preds,\n        'weights': weights\n    }\n\n# ===== Main Function =====\ndef main():\n    \"\"\"Complete enhanced pipeline execution\"\"\"\n    print(\"=\"*80)\n    print(\"ENHANCED CRYPTO PREDICTION PIPELINE WITH ANTI-OVERFITTING\")\n    print(\"=\"*80)\n    \n    # Load data\n    print(\"\\nLoading data...\")\n    train_df = pd.read_parquet(Config.TRAIN_PATH)\n    test_df = pd.read_parquet(Config.TEST_PATH)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    print(f\"Initial shapes - Train: {train_df.shape}, Test: {test_df.shape}\")\n    \n    # Feature selection with stability\n    print(\"\\n\" + \"=\"*80)\n    print(\"PHASE 1: Feature Selection with Stability Analysis\")\n    print(\"=\"*80)\n    \n    # Remove uninformative columns\n    train_clean, test_clean, kept_columns = remove_uninformative_columns(train_df, test_df)\n    \n    # Enhanced bucketing with noise\n    bucketer = EnhancedIntelligentBucketer(\n        min_buckets=Config.MIN_BUCKETS, \n        max_buckets=Config.MAX_BUCKETS,\n        noise_level=Config.NOISE_LEVEL\n    )\n    train_bucketed = bucketer.fit_transform(train_clean, kept_columns)\n    test_bucketed = bucketer.transform(test_clean, kept_columns)\n    \n    # Feature selection with stability\n    feature_columns = [col for col in kept_columns if col != Config.LABEL_COLUMN]\n    X_train = train_bucketed[feature_columns].values.astype(np.float32)\n    y_train = train_bucketed[Config.LABEL_COLUMN].values\n    \n    feature_importances = evaluate_features_with_stability(\n        X_train, y_train, feature_columns, n_runs=3\n    )\n    \n    # Select top features\n    sorted_features = sorted(feature_importances.items(), key=lambda x: x[1], reverse=True)\n    selected_features = [feat for feat, score in sorted_features[:Config.TOP_FEATURES_TO_SELECT]]\n    \n    # Ensure important market features are included\n    important_features = ['buy_qty', 'sell_qty', 'volume', 'bid_qty', 'ask_qty']\n    for feat in important_features:\n        if feat in feature_columns and feat not in selected_features:\n            selected_features.append(feat)\n    \n    print(f\"\\nSelected {len(selected_features)} features\")\n    print(\"Top 10 features:\")\n    for i, (feat, score) in enumerate(sorted_features[:10]):\n        print(f\"  {i+1:2d}. {feat:15s}: {score:.4f}\")\n    \n    # Train models with enhanced CV\n    print(\"\\n\" + \"=\"*80)\n    print(\"PHASE 2: Enhanced Model Training\")\n    print(\"=\"*80)\n    \n    results = train_with_enhanced_cv(train_bucketed, test_bucketed, selected_features)\n    \n    # Create submission\n    submission_df[\"prediction\"] = results['ensemble_test']\n    submission_df.to_csv(\"submission_enhanced.csv\", index=False)\n    print(\"\\nSaved: submission_enhanced.csv\")\n    \n    # Summary\n    print(\"\\n\" + \"=\"*80)\n    print(\"SUMMARY\")\n    print(\"=\"*80)\n    print(f\"Selected features: {len(selected_features)}\")\n    print(f\"Final ensemble score: {results['ensemble_score']:.4f}\")\n    print(f\"Number of models in ensemble: {len(results['all_test_preds'])}\")\n    \n    return selected_features, results\n\n# Execute\nif __name__ == \"__main__\":\n    selected_features, results = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}