{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr, rankdata, boxcox\nfrom scipy.special import lambertw\nfrom sklearn.preprocessing import QuantileTransformer, PowerTransformer\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# =========================\n# Configuration\n# =========================\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    FEATURES = [\n        \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\nLEARNERS = [\n    {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS}\n]\n\n# =========================\n# Helper Functions for Infinity Handling\n# =========================\ndef clip_extreme_values(x, percentile_range=(0.01, 99.99)):\n    \"\"\"Clip extreme values to specified percentile range\"\"\"\n    finite_mask = np.isfinite(x)\n    if not finite_mask.any():\n        return x\n    \n    finite_values = x[finite_mask]\n    lower = np.percentile(finite_values, percentile_range[0])\n    upper = np.percentile(finite_values, percentile_range[1])\n    \n    x_clipped = x.copy()\n    x_clipped = np.clip(x_clipped, lower, upper)\n    return x_clipped\n\ndef handle_infinities(x, method='clip'):\n    \"\"\"Replace infinite values with finite extremes\"\"\"\n    if not np.any(~np.isfinite(x)):\n        return x\n    \n    x_clean = x.copy()\n    finite_mask = np.isfinite(x)\n    \n    if finite_mask.any():\n        finite_values = x[finite_mask]\n        \n        if method == 'clip':\n            # Replace with extreme percentiles\n            min_val = np.percentile(finite_values, 0.1)\n            max_val = np.percentile(finite_values, 99.9)\n            range_val = max_val - min_val\n            \n            x_clean[x == float('-inf')] = min_val - 0.01 * range_val\n            x_clean[x == float('inf')] = max_val + 0.01 * range_val\n            x_clean[np.isnan(x)] = np.median(finite_values)\n        \n        elif method == 'median':\n            # Replace with median\n            median_val = np.median(finite_values)\n            x_clean[~finite_mask] = median_val\n    \n    return x_clean\n\ndef reduce_mem_usage(dataframe, dataset, verbose=True):    \n    \"\"\"Reduces memory usage by converting columns to more efficient data types\"\"\"\n    if verbose:\n        print(f'Reducing memory usage for: {dataset}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n\n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    if verbose:\n        print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n        print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n        print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n\n    return dataframe\n\n# =========================\n# Transformation Functions with Robust Infinity Handling\n# =========================\ndef robust_rank_transform(x, method='average'):\n    \"\"\"Rank transformation - extremely robust to outliers and noise\"\"\"\n    # Handle infinities first\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    ranks = np.zeros_like(x, dtype=float)\n    \n    if finite_mask.any():\n        finite_ranks = rankdata(x[finite_mask], method=method)\n        ranks[finite_mask] = finite_ranks\n        \n        if (~finite_mask).any():\n            max_rank = np.max(finite_ranks)\n            ranks[x == float('-inf')] = 0\n            ranks[x == float('inf')] = max_rank + 1\n    \n    if ranks.max() > ranks.min():\n        ranks = (ranks - ranks.min()) / (ranks.max() - ranks.min())\n    \n    return ranks\n\ndef robust_quantile_transform(x, n_quantiles=1000, subsample=100000):\n    \"\"\"Robust quantile transformation with subsampling for large datasets\"\"\"\n    # Handle infinities first\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    if len(x_finite) > subsample:\n        np.random.seed(42)\n        indices = np.random.choice(len(x_finite), subsample, replace=False)\n        fit_data = x_finite[indices]\n    else:\n        fit_data = x_finite\n    \n    qt = QuantileTransformer(n_quantiles=min(n_quantiles, len(fit_data)), \n                            output_distribution='normal')\n    qt.fit(fit_data.reshape(-1, 1))\n    \n    x_transformed = x.copy()\n    x_transformed[finite_mask] = qt.transform(x_finite.reshape(-1, 1)).ravel()\n    \n    # Post-transformation clip to prevent extreme values\n    x_transformed = clip_extreme_values(x_transformed, percentile_range=(0.1, 99.9))\n    \n    return x_transformed\n\ndef tapered_sigmoid_transform(x, k=0.1):\n    \"\"\"Apply a tapered sigmoid transformation\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    median = np.median(x_finite)\n    mad = np.median(np.abs(x_finite - median))\n    \n    if mad == 0:\n        mad = np.std(x_finite)\n    if mad == 0:\n        return x\n    \n    x_standardized = (x - median) / (mad * 1.4826)\n    x_standardized = np.clip(x_standardized, -10, 10)  # Prevent extreme values\n    \n    x_transformed = np.tanh(k * x_standardized)\n    \n    scale = np.percentile(x_finite, 95) - np.percentile(x_finite, 5)\n    x_final = x_transformed * scale / 2 + median\n    \n    return x_final\n\ndef sinh_arcsinh_transform(x, epsilon=0.1, delta=1.0):\n    \"\"\"Sinh-arcsinh transformation with infinity handling\"\"\"\n    # Handle infinities and clip extreme values\n    x = handle_infinities(x, method='clip')\n    x = clip_extreme_values(x, percentile_range=(0.1, 99.9))\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    mean = np.mean(x_finite)\n    std = np.std(x_finite)\n    if std == 0:\n        return x\n    \n    # Limit standardized values to prevent overflow\n    x_standardized = (x - mean) / std\n    x_standardized = np.clip(x_standardized, -10, 10)\n    \n    # Apply transformation with overflow protection\n    with np.errstate(over='ignore', invalid='ignore'):\n        x_transformed = np.sinh(delta * np.arcsinh(x_standardized) - epsilon)\n    \n    # Replace any new infinities\n    x_transformed = handle_infinities(x_transformed, method='clip')\n    \n    # Scale back\n    x_final = x_transformed * std + mean\n    \n    # Final clip to ensure no extreme values\n    x_final = clip_extreme_values(x_final, percentile_range=(0.05, 99.95))\n    \n    return x_final\n\ndef adaptive_scaling_transform(x, n_bins=20):\n    \"\"\"Adaptive scaling based on local density\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    # Create bins based on value distribution\n    percentiles = np.linspace(0, 100, n_bins + 1)\n    bin_edges = np.percentile(x_finite, percentiles)\n    bin_edges[0] -= 1e-10\n    bin_edges[-1] += 1e-10\n    \n    # Calculate scale for each bin\n    scales = np.ones_like(x_finite)\n    for i in range(n_bins):\n        mask = (x_finite >= bin_edges[i]) & (x_finite < bin_edges[i + 1])\n        if np.sum(mask) > 1:\n            bin_values = x_finite[mask]\n            median = np.median(bin_values)\n            mad = np.median(np.abs(bin_values - median))\n            scale = mad * 1.4826 if mad > 0 else np.std(bin_values)\n            scales[mask] = scale if scale > 0 else 1.0\n    \n    # Apply transformation\n    x_transformed = x.copy()\n    with np.errstate(divide='ignore', invalid='ignore'):\n        x_transformed[finite_mask] = x_finite / scales\n    \n    # Handle any new infinities\n    x_transformed = handle_infinities(x_transformed, method='clip')\n    \n    # Clip extreme values\n    x_transformed = clip_extreme_values(x_transformed)\n    \n    return x_transformed\n\ndef regime_detection_transform(x, n_regimes=5):\n    \"\"\"Regime detection based on distribution clustering\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    # Use percentile-based regime detection\n    percentiles = np.linspace(0, 100, n_regimes + 1)\n    regime_boundaries = np.percentile(x_finite, percentiles)\n    \n    # Encode values based on regime\n    x_transformed = x.copy()\n    finite_values = x_transformed[finite_mask]\n    \n    for i in range(len(finite_values)):\n        val = finite_values[i]\n        regime = np.searchsorted(regime_boundaries[1:-1], val)\n        finite_values[i] = regime / (n_regimes - 1)\n    \n    x_transformed[finite_mask] = finite_values\n    \n    # This transformation is bounded [0, 1] so no additional handling needed\n    return x_transformed\n\ndef modified_zscore_transform(x, threshold=3.5):\n    \"\"\"Modified Z-score using median and MAD for robustness\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    median = np.median(x_finite)\n    mad = np.median(np.abs(x_finite - median))\n    \n    if mad == 0:\n        mad = np.std(x_finite)\n    if mad == 0:\n        return x\n    \n    # Modified z-score\n    with np.errstate(divide='ignore', invalid='ignore'):\n        modified_z = 0.6745 * (x - median) / mad\n    \n    # Clip extreme values\n    x_transformed = np.clip(modified_z, -threshold, threshold)\n    \n    # Handle any infinities\n    x_transformed = handle_infinities(x_transformed, method='clip')\n    \n    return x_transformed\n\ndef asymmetric_winsorize_transform(x, lower_pct=1, upper_pct=5):\n    \"\"\"Asymmetric winsorization adapted for crypto (positive skew)\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    lower_bound = np.percentile(x_finite, lower_pct)\n    upper_bound = np.percentile(x_finite, 100 - upper_pct)\n    \n    x_transformed = x.copy()\n    x_transformed[x_transformed < lower_bound] = lower_bound\n    x_transformed[x_transformed > upper_bound] = upper_bound\n    \n    return x_transformed\n\ndef log_transform_robust(x, shift_method='auto'):\n    \"\"\"Robust log transformation with automatic shift for negative values\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    # Determine shift\n    min_val = np.min(x_finite)\n    if shift_method == 'auto':\n        if min_val <= 0:\n            shift = abs(min_val) + 1\n        else:\n            shift = 0\n    else:\n        shift = shift_method\n    \n    # Apply log transformation\n    x_transformed = x.copy()\n    with np.errstate(divide='ignore', invalid='ignore'):\n        x_transformed[finite_mask] = np.log1p(x_finite + shift)\n    \n    # Handle any new infinities\n    x_transformed = handle_infinities(x_transformed, method='clip')\n    \n    return x_transformed\n\ndef huber_transform(x, epsilon=1.345):\n    \"\"\"Huber transformation - robust to outliers\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    # Standardize\n    median = np.median(x_finite)\n    mad = np.median(np.abs(x_finite - median))\n    \n    if mad == 0:\n        mad = np.std(x_finite)\n    if mad == 0:\n        return x\n    \n    x_standardized = (x - median) / (mad * 1.4826)\n    x_standardized = np.clip(x_standardized, -20, 20)  # Prevent extreme values\n    \n    # Huber transformation\n    x_transformed = x.copy()\n    mask_linear = np.abs(x_standardized) <= epsilon\n    mask_log = ~mask_linear & finite_mask\n    \n    x_transformed[mask_linear] = x_standardized[mask_linear]\n    \n    with np.errstate(divide='ignore', invalid='ignore'):\n        x_transformed[mask_log] = epsilon * np.sign(x_standardized[mask_log]) * \\\n                                  (1 + np.log(np.abs(x_standardized[mask_log]) / epsilon))\n    \n    # Handle any infinities\n    x_transformed = handle_infinities(x_transformed, method='clip')\n    \n    # Scale back\n    scale = np.percentile(x_finite, 95) - np.percentile(x_finite, 5)\n    x_transformed = x_transformed * scale / (2 * epsilon) + median\n    \n    return x_transformed\n\ndef box_cox_robust(x):\n    \"\"\"Box-Cox transformation with robust handling\"\"\"\n    # Handle infinities and clip extreme values\n    x = handle_infinities(x, method='clip')\n    x = clip_extreme_values(x, percentile_range=(0.1, 99.9))\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    # Ensure positive values\n    min_val = np.min(x_finite)\n    if min_val <= 0:\n        shift = abs(min_val) + 1\n        x_shifted = x_finite + shift\n    else:\n        shift = 0\n        x_shifted = x_finite\n    \n    try:\n        # Clip to reasonable range before Box-Cox\n        x_shifted = np.clip(x_shifted, 1e-10, 1e10)\n        \n        x_bc, lambda_param = boxcox(x_shifted)\n        \n        # Check for infinities in result\n        if not np.isfinite(x_bc).all():\n            raise ValueError(\"Box-Cox produced non-finite values\")\n        \n        x_transformed = x.copy()\n        x_transformed[finite_mask] = x_bc\n        \n        # Normalize to original scale\n        bc_mean = np.mean(x_bc)\n        bc_std = np.std(x_bc)\n        if bc_std > 0:\n            x_transformed[finite_mask] = (x_transformed[finite_mask] - bc_mean) * \\\n                                        np.std(x_finite) / bc_std + np.mean(x_finite)\n        \n        # Final clip\n        x_transformed = clip_extreme_values(x_transformed)\n        \n    except:\n        # Fallback to log transform if Box-Cox fails\n        x_transformed = log_transform_robust(x)\n    \n    return x_transformed\n\ndef percentile_rank_transform(x):\n    \"\"\"Percentile rank transformation - converts values to their percentile ranks\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    from scipy.stats import percentileofscore\n    \n    x_transformed = x.copy()\n    percentiles = np.zeros_like(x_finite)\n    \n    # Use vectorized approach for efficiency\n    sorted_indices = np.argsort(x_finite)\n    percentiles[sorted_indices] = np.arange(1, len(x_finite) + 1) / len(x_finite)\n    \n    x_transformed[finite_mask] = percentiles\n    \n    # This transformation is bounded [0, 1] so no additional handling needed\n    return x_transformed\n\ndef double_sigmoid_transform(x, k1=0.1, k2=0.05):\n    \"\"\"Double sigmoid - different transformations for positive and negative values\"\"\"\n    # Handle infinities\n    x = handle_infinities(x, method='clip')\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    median = np.median(x_finite)\n    mad = np.median(np.abs(x_finite - median))\n    \n    if mad == 0:\n        mad = np.std(x_finite)\n    if mad == 0:\n        return x\n    \n    x_standardized = (x - median) / (mad * 1.4826)\n    x_standardized = np.clip(x_standardized, -10, 10)  # Prevent extreme values\n    \n    # Apply different sigmoid for positive and negative values\n    x_transformed = x.copy()\n    pos_mask = (x_standardized >= 0) & finite_mask\n    neg_mask = (x_standardized < 0) & finite_mask\n    \n    x_transformed[pos_mask] = np.tanh(k1 * x_standardized[pos_mask])\n    x_transformed[neg_mask] = np.tanh(k2 * x_standardized[neg_mask])\n    \n    # Scale back\n    scale = np.percentile(x_finite, 95) - np.percentile(x_finite, 5)\n    x_transformed = x_transformed * scale / 2 + median\n    \n    return x_transformed\n\ndef lambert_w_transform(x):\n    \"\"\"Lambert W transformation with robust handling\"\"\"\n    # Handle infinities and clip extreme values\n    x = handle_infinities(x, method='clip')\n    x = clip_extreme_values(x, percentile_range=(0.5, 99.5))\n    \n    finite_mask = np.isfinite(x)\n    x_finite = x[finite_mask]\n    \n    if len(x_finite) == 0:\n        return x\n    \n    # Standardize\n    mean = np.mean(x_finite)\n    std = np.std(x_finite)\n    if std == 0:\n        return x\n    \n    x_standardized = (x - mean) / std\n    x_standardized = np.clip(x_standardized, -5, 5)  # Limit range\n    \n    # Estimate delta parameter using skewness\n    from scipy.stats import skew\n    skewness = skew(x_finite)\n    delta = np.sign(skewness) * min(abs(skewness) / 3, 0.3)  # Reduced delta\n    \n    # Apply Lambert W transformation\n    x_transformed = x.copy()\n    u = delta * x_standardized\n    \n    # For numerical stability\n    mask_small = np.abs(u) < 0.01\n    x_transformed[mask_small & finite_mask] = x_standardized[mask_small & finite_mask]\n    \n    mask_large = ~mask_small & finite_mask\n    if mask_large.any():\n        u_large = u[mask_large & finite_mask]\n        # Limit the argument to lambertw\n        arg = np.clip(u_large * np.exp(u_large), -700, 700)\n        \n        with np.errstate(invalid='ignore', over='ignore'):\n            w_values = np.real(lambertw(arg))\n        \n        # Replace any infinities\n        w_values = handle_infinities(w_values, method='clip')\n        \n        if abs(delta) > 1e-10:\n            x_transformed[mask_large] = w_values / delta\n        else:\n            x_transformed[mask_large] = x_standardized[mask_large]\n    \n    # Scale back\n    x_transformed = x_transformed * std + mean\n    \n    # Final clip\n    x_transformed = clip_extreme_values(x_transformed)\n    \n    return x_transformed\n\ndef apply_transformation(df, method, columns=None, **kwargs):\n    \"\"\"Apply transformation to specified columns with robust infinity handling\"\"\"\n    transformed_df = df.copy()\n    \n    if columns is None:\n        numeric_cols = df.select_dtypes(include=[np.number]).columns\n        columns = [col for col in numeric_cols if col not in ['timestamp', 'label']]\n    \n    for col in columns:\n        original_values = df[col].values.copy()\n        \n        # Pre-process: handle existing infinities\n        original_values = handle_infinities(original_values, method='clip')\n        \n        # Apply transformation\n        if method == 'sigmoid':\n            k = kwargs.get('k', 0.1)\n            transformed_values = tapered_sigmoid_transform(original_values, k=k)\n            \n        elif method == 'rank':\n            rank_method = kwargs.get('rank_method', 'average')\n            transformed_values = robust_rank_transform(original_values, method=rank_method)\n            \n        elif method == 'robust_quantile':\n            n_quantiles = kwargs.get('n_quantiles', 1000)\n            subsample = kwargs.get('subsample', 100000)\n            transformed_values = robust_quantile_transform(original_values, \n                                                         n_quantiles=n_quantiles,\n                                                         subsample=subsample)\n            \n        elif method == 'sinh_arcsinh':\n            epsilon = kwargs.get('epsilon', 0.1)\n            delta = kwargs.get('delta', 1.0)\n            transformed_values = sinh_arcsinh_transform(original_values, epsilon=epsilon, delta=delta)\n            \n        elif method == 'yeo_johnson':\n            try:\n                # Clip extreme values before transformation\n                clipped_values = clip_extreme_values(original_values, percentile_range=(0.1, 99.9))\n                pt = PowerTransformer(method='yeo-johnson', standardize=False)\n                transformed_values = pt.fit_transform(clipped_values.reshape(-1, 1)).ravel()\n                # Clip again after transformation\n                transformed_values = clip_extreme_values(transformed_values)\n            except:\n                # Fallback to rank transform\n                transformed_values = robust_rank_transform(original_values)\n            \n        elif method == 'quantile':\n            try:\n                output_dist = kwargs.get('output_distribution', 'normal')\n                n_quantiles = kwargs.get('n_quantiles', 10000)\n                \n                # Use robust quantile transform instead\n                transformed_values = robust_quantile_transform(original_values, \n                                                             n_quantiles=n_quantiles,\n                                                             subsample=100000)\n                \n                if output_dist == 'normal':\n                    finite_mask = np.isfinite(original_values)\n                    if finite_mask.any():\n                        scale = np.std(original_values[finite_mask])\n                        center = np.median(original_values[finite_mask])\n                        transformed_values = transformed_values * scale + center\n                        transformed_values = clip_extreme_values(transformed_values)\n            except:\n                # Fallback\n                transformed_values = robust_rank_transform(original_values)\n                \n        elif method == 'adaptive_scaling':\n            n_bins = kwargs.get('n_bins', 20)\n            transformed_values = adaptive_scaling_transform(original_values, n_bins=n_bins)\n            \n        elif method == 'regime_detection':\n            n_regimes = kwargs.get('n_regimes', 5)\n            transformed_values = regime_detection_transform(original_values, n_regimes=n_regimes)\n            \n        elif method == 'modified_zscore':\n            threshold = kwargs.get('threshold', 3.5)\n            transformed_values = modified_zscore_transform(original_values, threshold=threshold)\n            \n        elif method == 'asymmetric_winsorize':\n            lower_pct = kwargs.get('lower_pct', 1)\n            upper_pct = kwargs.get('upper_pct', 5)\n            transformed_values = asymmetric_winsorize_transform(original_values, \n                                                              lower_pct=lower_pct, \n                                                              upper_pct=upper_pct)\n            \n        elif method == 'log_robust':\n            shift_method = kwargs.get('shift_method', 'auto')\n            transformed_values = log_transform_robust(original_values, shift_method=shift_method)\n            \n        elif method == 'huber':\n            epsilon = kwargs.get('epsilon', 1.345)\n            transformed_values = huber_transform(original_values, epsilon=epsilon)\n            \n        elif method == 'box_cox':\n            transformed_values = box_cox_robust(original_values)\n            \n        elif method == 'lambert_w':\n            transformed_values = lambert_w_transform(original_values)\n            \n        elif method == 'percentile_rank':\n            transformed_values = percentile_rank_transform(original_values)\n            \n        elif method == 'double_sigmoid':\n            k1 = kwargs.get('k1', 0.1)\n            k2 = kwargs.get('k2', 0.05)\n            transformed_values = double_sigmoid_transform(original_values, k1=k1, k2=k2)\n        \n        else:\n            raise ValueError(f\"Unknown transformation method: {method}\")\n        \n        # Post-process: final check for infinities\n        transformed_values = handle_infinities(transformed_values, method='clip')\n        \n        # Ensure no extreme values remain\n        if method not in ['rank', 'regime_detection', 'percentile_rank']:  # These are already bounded\n            transformed_values = clip_extreme_values(transformed_values, percentile_range=(0.01, 99.99))\n        \n        transformed_df[col] = transformed_values\n    \n    return transformed_df\n\n# =========================\n# Training Functions\n# =========================\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef get_model_slices(n_samples: int):\n    return [\n        {\"name\": \"full_data\", \"cutoff\": 0},\n        {\"name\": \"last_75pct\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_50pct\", \"cutoff\": int(0.50 * n_samples)}\n    ]\n\ndef train_and_evaluate(train_df, test_df):\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS\n    }\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"  Fold {fold}/{Config.N_FOLDS}\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            subset = train_df.iloc[cutoff:].reset_index(drop=True)\n            rel_idx = train_idx[train_idx >= cutoff] - cutoff\n\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n            for learner in LEARNERS:\n                model = learner[\"Estimator\"](**learner[\"params\"])\n                model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n                mask = valid_idx >= cutoff\n                if mask.any():\n                    idxs = valid_idx[mask]\n                    oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n                if cutoff > 0 and (~mask).any():\n                    oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n\n                test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[Config.FEATURES])\n\n    # Normalize test predictions\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= Config.N_FOLDS\n\n    return oof_preds, test_preds\n\ndef create_submission(train_df, oof_preds, test_preds, submission_df, method_name):\n    learner_ensembles = {}\n    for learner_name in oof_preds:\n        scores = {s: pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[learner_name][s])[0]\n                  for s in oof_preds[learner_name]}\n        total_score = sum(scores.values())\n\n        oof_simple = np.mean(list(oof_preds[learner_name].values()), axis=0)\n        test_simple = np.mean(list(test_preds[learner_name].values()), axis=0)\n        score_simple = pearsonr(train_df[Config.LABEL_COLUMN], oof_simple)[0]\n\n        oof_weighted = sum(scores[s] / total_score * oof_preds[learner_name][s] for s in scores)\n        test_weighted = sum(scores[s] / total_score * test_preds[learner_name][s] for s in scores)\n        score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], oof_weighted)[0]\n\n        print(f\"  {learner_name.upper()} Simple Ensemble Pearson:   {score_simple:.4f}\")\n        print(f\"  {learner_name.upper()} Weighted Ensemble Pearson: {score_weighted:.4f}\")\n\n        learner_ensembles[learner_name] = {\n            \"oof_simple\": oof_simple,\n            \"test_simple\": test_simple,\n            \"oof_weighted\": oof_weighted,\n            \"test_weighted\": test_weighted\n        }\n\n    final_oof = np.mean([le[\"oof_weighted\"] for le in learner_ensembles.values()], axis=0)\n    final_test = np.mean([le[\"test_weighted\"] for le in learner_ensembles.values()], axis=0)\n    final_score = pearsonr(train_df[Config.LABEL_COLUMN], final_oof)[0]\n\n    print(f\"  FINAL ensemble Pearson: {final_score:.4f}\")\n\n    submission_df[\"prediction\"] = final_test\n    filename = f\"submission_{method_name}.csv\"\n    submission_df.to_csv(filename, index=False)\n    print(f\"  Saved: {filename}\")\n    \n    return final_score\n\n# =========================\n# Main Execution\n# =========================\ndef main():\n    # Define all transformation methods with their parameters\n    transformation_methods = [\n        # Original methods (with safer parameters)\n        {\"name\": \"robust_quantile\", \"params\": {\"n_quantiles\": 1000, \"subsample\": 100000}},\n        {\"name\": \"rank\", \"params\": {\"rank_method\": \"average\"}},\n        {\"name\": \"sigmoid\", \"params\": {\"k\": 0.05}},\n        {\"name\": \"quantile\", \"params\": {\"output_distribution\": \"normal\", \"n_quantiles\": 10000}},\n        {\"name\": \"sinh_arcsinh\", \"params\": {\"epsilon\": 0.1, \"delta\": 0.5}},  # Reduced delta\n        {\"name\": \"yeo_johnson\", \"params\": {}},\n        \n        # New distribution-based methods\n        {\"name\": \"adaptive_scaling\", \"params\": {\"n_bins\": 20}},\n        {\"name\": \"regime_detection\", \"params\": {\"n_regimes\": 5}},\n        {\"name\": \"modified_zscore\", \"params\": {\"threshold\": 3.5}},\n        {\"name\": \"asymmetric_winsorize\", \"params\": {\"lower_pct\": 1, \"upper_pct\": 5}},\n        {\"name\": \"log_robust\", \"params\": {\"shift_method\": \"auto\"}},\n        {\"name\": \"huber\", \"params\": {\"epsilon\": 1.345}},\n        {\"name\": \"box_cox\", \"params\": {}},\n        {\"name\": \"lambert_w\", \"params\": {}},\n        {\"name\": \"percentile_rank\", \"params\": {}},\n        {\"name\": \"double_sigmoid\", \"params\": {\"k1\": 0.1, \"k2\": 0.05}}\n    ]\n    \n    # Load raw data once\n    print(\"Loading raw data...\")\n    train_df_raw = pd.read_parquet(Config.TRAIN_PATH)\n    test_df_raw = pd.read_parquet(Config.TEST_PATH)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    print(f\"Raw data - Train: {train_df_raw.shape}, Test: {test_df_raw.shape}\")\n    \n    # Check for infinities in raw data\n    print(\"\\nChecking for infinities in raw data...\")\n    for col in Config.FEATURES:\n        n_inf_train = (~np.isfinite(train_df_raw[col])).sum()\n        n_inf_test = (~np.isfinite(test_df_raw[col])).sum()\n        if n_inf_train > 0 or n_inf_test > 0:\n            print(f\"  {col}: Train infinities = {n_inf_train}, Test infinities = {n_inf_test}\")\n    \n    # Reduce memory usage\n    train_df_raw = reduce_mem_usage(train_df_raw, 'train', verbose=False)\n    test_df_raw = reduce_mem_usage(test_df_raw, 'test', verbose=False)\n    \n    results = []\n    \n    # Process each transformation method\n    for method_info in transformation_methods:\n        method_name = method_info[\"name\"]\n        method_params = method_info[\"params\"]\n        \n        print(f\"\\n{'='*60}\")\n        print(f\"Processing transformation method: {method_name.upper()}\")\n        print(f\"{'='*60}\")\n        \n        try:\n            # Apply transformation only to FEATURES columns\n            print(f\"Applying {method_name} transformation...\")\n            train_df = train_df_raw.copy()\n            test_df = test_df_raw.copy()\n            \n            # Transform only the feature columns\n            train_df[Config.FEATURES] = apply_transformation(\n                train_df[Config.FEATURES], \n                method_name, \n                columns=Config.FEATURES,\n                **method_params\n            )[Config.FEATURES]\n            \n            test_df[Config.FEATURES] = apply_transformation(\n                test_df[Config.FEATURES], \n                method_name, \n                columns=Config.FEATURES,\n                **method_params\n            )[Config.FEATURES]\n            \n            # Verify no infinities remain\n            has_inf = False\n            for col in Config.FEATURES:\n                if (~np.isfinite(train_df[col])).any() or (~np.isfinite(test_df[col])).any():\n                    print(f\"  WARNING: Column {col} still has infinities after transformation!\")\n                    has_inf = True\n            \n            if has_inf:\n                print(f\"  Skipping {method_name} due to remaining infinities\")\n                continue\n            \n            # Reset index\n            train_df = train_df.reset_index(drop=True)\n            test_df = test_df.reset_index(drop=True)\n            \n            # Train and evaluate\n            print(f\"Training models with {method_name} transformation...\")\n            oof_preds, test_preds = train_and_evaluate(train_df, test_df)\n            \n            # Create submission\n            print(f\"Creating submission for {method_name}...\")\n            final_score = create_submission(train_df, oof_preds, test_preds, \n                                          submission_df.copy(), method_name)\n            \n            results.append({\"method\": method_name, \"score\": final_score})\n            \n        except Exception as e:\n            print(f\"  ERROR with {method_name}: {str(e)}\")\n            import traceback\n            traceback.print_exc()\n            continue\n    \n    # Summary\n    print(f\"\\n{'='*60}\")\n    print(\"SUMMARY OF ALL METHODS\")\n    print(f\"{'='*60}\")\n    for result in sorted(results, key=lambda x: x['score'], reverse=True):\n        print(f\"{result['method']:<25} Pearson: {result['score']:.4f}\")\n    \n    if results:\n        best_method = max(results, key=lambda x: x['score'])\n        print(f\"\\nBest method: {best_method['method']} (Pearson: {best_method['score']:.4f})\")\n        print(f\"Total submission files created: {len(results)}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import sys\n# import pandas as pd\n# import numpy as np\n# from sklearn.model_selection import KFold\n# from xgboost import XGBRegressor\n# from lightgbm import LGBMRegressor\n# from scipy.stats import pearsonr, rankdata, boxcox\n# from scipy.special import lambertw\n# from sklearn.preprocessing import QuantileTransformer, PowerTransformer\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # =========================\n# # Configuration\n# # =========================\n# class Config:\n#     TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n#     TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n#     SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n#     FEATURES = [\n#         \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n#         \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n#         \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\"\n#     ]\n\n#     LABEL_COLUMN = \"label\"\n#     N_FOLDS = 3\n#     RANDOM_STATE = 42\n\n# XGB_PARAMS = {\n#     \"tree_method\": \"hist\",\n#     \"device\": \"gpu\",\n#     \"colsample_bylevel\": 0.4778,\n#     \"colsample_bynode\": 0.3628,\n#     \"colsample_bytree\": 0.7107,\n#     \"gamma\": 1.7095,\n#     \"learning_rate\": 0.02213,\n#     \"max_depth\": 20,\n#     \"max_leaves\": 12,\n#     \"min_child_weight\": 16,\n#     \"n_estimators\": 1667,\n#     \"subsample\": 0.06567,\n#     \"reg_alpha\": 39.3524,\n#     \"reg_lambda\": 75.4484,\n#     \"verbosity\": 0,\n#     \"random_state\": Config.RANDOM_STATE,\n#     \"n_jobs\": -1\n# }\n\n# LEARNERS = [\n#     {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS}\n# ]\n\n# # =========================\n# # Helper Functions\n# # =========================\n# def clip_extreme_values(x, percentile_range=(0.01, 99.99)):\n#     \"\"\"Clip extreme values to specified percentile range\"\"\"\n#     finite_mask = np.isfinite(x)\n#     if not finite_mask.any():\n#         return x\n    \n#     finite_values = x[finite_mask]\n#     lower = np.percentile(finite_values, percentile_range[0])\n#     upper = np.percentile(finite_values, percentile_range[1])\n    \n#     x_clipped = x.copy()\n#     x_clipped = np.clip(x_clipped, lower, upper)\n#     return x_clipped\n\n# def handle_infinities(x, method='clip'):\n#     \"\"\"Replace infinite values with finite extremes\"\"\"\n#     if not np.any(~np.isfinite(x)):\n#         return x\n    \n#     x_clean = x.copy()\n#     finite_mask = np.isfinite(x)\n    \n#     if finite_mask.any():\n#         finite_values = x[finite_mask]\n        \n#         if method == 'clip':\n#             # Replace with extreme percentiles\n#             min_val = np.percentile(finite_values, 0.1)\n#             max_val = np.percentile(finite_values, 99.9)\n#             range_val = max_val - min_val\n            \n#             x_clean[x == float('-inf')] = min_val - 0.01 * range_val\n#             x_clean[x == float('inf')] = max_val + 0.01 * range_val\n#             x_clean[np.isnan(x)] = np.median(finite_values)\n        \n#         elif method == 'median':\n#             # Replace with median\n#             median_val = np.median(finite_values)\n#             x_clean[~finite_mask] = median_val\n    \n#     return x_clean\n\n# def reduce_mem_usage(dataframe, dataset, verbose=True):    \n#     \"\"\"Reduces memory usage by converting columns to more efficient data types\"\"\"\n#     if verbose:\n#         print(f'Reducing memory usage for: {dataset}')\n#     initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n\n#     for col in dataframe.columns:\n#         col_type = dataframe[col].dtype\n\n#         c_min = dataframe[col].min()\n#         c_max = dataframe[col].max()\n#         if str(col_type)[:3] == 'int':\n#             if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n#                 dataframe[col] = dataframe[col].astype(np.int8)\n#             elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n#                 dataframe[col] = dataframe[col].astype(np.int16)\n#             elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n#                 dataframe[col] = dataframe[col].astype(np.int32)\n#             elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n#                 dataframe[col] = dataframe[col].astype(np.int64)\n#         else:\n#             if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n#                 dataframe[col] = dataframe[col].astype(np.float16)\n#             elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n#                 dataframe[col] = dataframe[col].astype(np.float32)\n#             else:\n#                 dataframe[col] = dataframe[col].astype(np.float64)\n\n#     final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n#     if verbose:\n#         print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n#         print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n#         print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n\n#     return dataframe\n\n# # =========================\n# # Transformation Functions (with infinity handling)\n# # =========================\n# def robust_rank_transform(x, method='average'):\n#     \"\"\"Rank transformation - extremely robust to outliers and noise\"\"\"\n#     # First handle infinities\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     ranks = np.zeros_like(x, dtype=float)\n    \n#     if finite_mask.any():\n#         finite_ranks = rankdata(x[finite_mask], method=method)\n#         ranks[finite_mask] = finite_ranks\n        \n#         if (~finite_mask).any():\n#             max_rank = np.max(finite_ranks)\n#             ranks[x == float('-inf')] = 0\n#             ranks[x == float('inf')] = max_rank + 1\n    \n#     if ranks.max() > ranks.min():\n#         ranks = (ranks - ranks.min()) / (ranks.max() - ranks.min())\n    \n#     return ranks\n\n# def robust_quantile_transform(x, n_quantiles=1000, subsample=100000):\n#     \"\"\"Robust quantile transformation with subsampling for large datasets\"\"\"\n#     # Handle infinities first\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     if len(x_finite) > subsample:\n#         np.random.seed(42)\n#         indices = np.random.choice(len(x_finite), subsample, replace=False)\n#         fit_data = x_finite[indices]\n#     else:\n#         fit_data = x_finite\n    \n#     qt = QuantileTransformer(n_quantiles=min(n_quantiles, len(fit_data)), \n#                             output_distribution='normal')\n#     qt.fit(fit_data.reshape(-1, 1))\n    \n#     x_transformed = x.copy()\n#     x_transformed[finite_mask] = qt.transform(x_finite.reshape(-1, 1)).ravel()\n    \n#     # Post-transformation clip to prevent extreme values\n#     x_transformed = clip_extreme_values(x_transformed, percentile_range=(0.1, 99.9))\n    \n#     return x_transformed\n\n# def tapered_sigmoid_transform(x, k=0.1):\n#     \"\"\"Apply a tapered sigmoid transformation\"\"\"\n#     # Handle infinities\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     median = np.median(x_finite)\n#     mad = np.median(np.abs(x_finite - median))\n    \n#     if mad == 0:\n#         mad = np.std(x_finite)\n#     if mad == 0:\n#         return x\n    \n#     x_standardized = (x - median) / (mad * 1.4826)\n#     x_transformed = np.tanh(k * x_standardized)\n    \n#     scale = np.percentile(x_finite, 95) - np.percentile(x_finite, 5)\n#     x_final = x_transformed * scale / 2 + median\n    \n#     return x_final\n\n# def sinh_arcsinh_transform(x, epsilon=0.1, delta=1.0):\n#     \"\"\"Sinh-arcsinh transformation with infinity handling\"\"\"\n#     # Handle infinities and clip extreme values\n#     x = handle_infinities(x, method='clip')\n#     x = clip_extreme_values(x, percentile_range=(0.1, 99.9))\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     mean = np.mean(x_finite)\n#     std = np.std(x_finite)\n#     if std == 0:\n#         return x\n    \n#     # Limit standardized values to prevent overflow\n#     x_standardized = (x - mean) / std\n#     x_standardized = np.clip(x_standardized, -10, 10)\n    \n#     # Apply transformation with overflow protection\n#     with np.errstate(over='ignore', invalid='ignore'):\n#         x_transformed = np.sinh(delta * np.arcsinh(x_standardized) - epsilon)\n    \n#     # Replace any new infinities\n#     x_transformed = handle_infinities(x_transformed, method='clip')\n    \n#     # Scale back\n#     x_final = x_transformed * std + mean\n    \n#     # Final clip to ensure no extreme values\n#     x_final = clip_extreme_values(x_final, percentile_range=(0.05, 99.95))\n    \n#     return x_final\n\n# def novas_transform(x, window_size=100):\n#     \"\"\"NoVaS transformation with infinity handling\"\"\"\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     sorted_indices = np.argsort(x_finite)\n#     sorted_x = x_finite[sorted_indices]\n    \n#     scales = np.zeros_like(sorted_x)\n#     for i in range(len(sorted_x)):\n#         start = max(0, i - window_size // 2)\n#         end = min(len(sorted_x), i + window_size // 2)\n#         window = sorted_x[start:end]\n        \n#         median = np.median(window)\n#         mad = np.median(np.abs(window - median))\n#         scales[i] = mad * 1.4826 if mad > 0 else 1.0\n    \n#     scale_map = np.zeros_like(x_finite)\n#     scale_map[sorted_indices] = scales\n    \n#     x_transformed = x.copy()\n#     x_transformed[finite_mask] = x_finite / scale_map\n    \n#     # Clip extreme values\n#     x_transformed = clip_extreme_values(x_transformed)\n    \n#     return x_transformed\n\n# def directional_change_transform(x, threshold=0.01):\n#     \"\"\"Directional Change framework adapted for feature transformation\"\"\"\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     sorted_indices = np.argsort(x_finite)\n#     sorted_x = x_finite[sorted_indices]\n    \n#     dc_levels = [sorted_x[0]]\n#     last_extreme = sorted_x[0]\n    \n#     for val in sorted_x[1:]:\n#         if abs(val - last_extreme) / (abs(last_extreme) + 1e-10) > threshold:\n#             dc_levels.append(val)\n#             last_extreme = val\n    \n#     dc_levels = np.array(dc_levels)\n    \n#     x_transformed = x.copy()\n#     finite_values = x_transformed[finite_mask]\n    \n#     for i, val in enumerate(finite_values):\n#         distances = np.abs(dc_levels - val)\n#         nearest_idx = np.argmin(distances)\n#         finite_values[i] = nearest_idx / (len(dc_levels) - 1) if len(dc_levels) > 1 else 0.5\n    \n#     x_transformed[finite_mask] = finite_values\n    \n#     if (x == float('-inf')).any():\n#         x_transformed[x == float('-inf')] = 0\n#     if (x == float('inf')).any():\n#         x_transformed[x == float('inf')] = 1\n    \n#     return x_transformed\n\n# def modified_zscore_transform(x, threshold=3.5):\n#     \"\"\"Modified Z-score using median and MAD for robustness\"\"\"\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     median = np.median(x_finite)\n#     mad = np.median(np.abs(x_finite - median))\n    \n#     if mad == 0:\n#         mad = np.std(x_finite)\n#     if mad == 0:\n#         return x\n    \n#     modified_z = 0.6745 * (x - median) / mad\n#     x_transformed = np.clip(modified_z, -threshold, threshold)\n    \n#     return x_transformed\n\n# def asymmetric_winsorize_transform(x, lower_pct=1, upper_pct=5):\n#     \"\"\"Asymmetric winsorization adapted for crypto (positive skew)\"\"\"\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     lower_bound = np.percentile(x_finite, lower_pct)\n#     upper_bound = np.percentile(x_finite, 100 - upper_pct)\n    \n#     x_transformed = x.copy()\n#     x_transformed[x_transformed < lower_bound] = lower_bound\n#     x_transformed[x_transformed > upper_bound] = upper_bound\n    \n#     return x_transformed\n\n# def log_transform_robust(x, shift_method='auto'):\n#     \"\"\"Robust log transformation with automatic shift for negative values\"\"\"\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     min_val = np.min(x_finite)\n#     if shift_method == 'auto':\n#         if min_val <= 0:\n#             shift = abs(min_val) + 1\n#         else:\n#             shift = 0\n#     else:\n#         shift = shift_method\n    \n#     x_transformed = x.copy()\n#     with np.errstate(divide='ignore', invalid='ignore'):\n#         x_transformed[finite_mask] = np.log1p(x_finite + shift)\n    \n#     # Handle any new infinities\n#     x_transformed = handle_infinities(x_transformed, method='clip')\n    \n#     return x_transformed\n\n# def huber_transform(x, epsilon=1.345):\n#     \"\"\"Huber transformation - robust to outliers\"\"\"\n#     x = handle_infinities(x, method='clip')\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     median = np.median(x_finite)\n#     mad = np.median(np.abs(x_finite - median))\n    \n#     if mad == 0:\n#         mad = np.std(x_finite)\n#     if mad == 0:\n#         return x\n    \n#     x_standardized = (x - median) / (mad * 1.4826)\n    \n#     x_transformed = x.copy()\n#     mask_linear = np.abs(x_standardized) <= epsilon\n#     mask_log = ~mask_linear & finite_mask\n    \n#     x_transformed[mask_linear] = x_standardized[mask_linear]\n    \n#     with np.errstate(divide='ignore', invalid='ignore'):\n#         x_transformed[mask_log] = epsilon * np.sign(x_standardized[mask_log]) * \\\n#                                   (1 + np.log(np.abs(x_standardized[mask_log]) / epsilon))\n    \n#     # Handle any infinities\n#     x_transformed = handle_infinities(x_transformed, method='clip')\n    \n#     scale = np.percentile(x_finite, 95) - np.percentile(x_finite, 5)\n#     x_transformed = x_transformed * scale / (2 * epsilon) + median\n    \n#     return x_transformed\n\n# def box_cox_robust(x):\n#     \"\"\"Box-Cox transformation with robust handling\"\"\"\n#     x = handle_infinities(x, method='clip')\n#     x = clip_extreme_values(x, percentile_range=(0.1, 99.9))\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     # Ensure positive values\n#     min_val = np.min(x_finite)\n#     if min_val <= 0:\n#         shift = abs(min_val) + 1\n#         x_shifted = x_finite + shift\n#     else:\n#         shift = 0\n#         x_shifted = x_finite\n    \n#     try:\n#         # Clip to reasonable range before Box-Cox\n#         x_shifted = np.clip(x_shifted, 1e-10, 1e10)\n        \n#         x_bc, lambda_param = boxcox(x_shifted)\n        \n#         # Check for infinities in result\n#         if not np.isfinite(x_bc).all():\n#             raise ValueError(\"Box-Cox produced non-finite values\")\n        \n#         x_transformed = x.copy()\n#         x_transformed[finite_mask] = x_bc\n        \n#         # Normalize to original scale\n#         bc_mean = np.mean(x_bc)\n#         bc_std = np.std(x_bc)\n#         if bc_std > 0:\n#             x_transformed[finite_mask] = (x_transformed[finite_mask] - bc_mean) * \\\n#                                         np.std(x_finite) / bc_std + np.mean(x_finite)\n        \n#         # Final clip\n#         x_transformed = clip_extreme_values(x_transformed)\n        \n#     except:\n#         # Fallback to log transform if Box-Cox fails\n#         x_transformed = log_transform_robust(x)\n    \n#     return x_transformed\n\n# def lambert_w_transform(x):\n#     \"\"\"Lambert W transformation with robust handling\"\"\"\n#     x = handle_infinities(x, method='clip')\n#     x = clip_extreme_values(x, percentile_range=(0.5, 99.5))\n    \n#     finite_mask = np.isfinite(x)\n#     x_finite = x[finite_mask]\n    \n#     if len(x_finite) == 0:\n#         return x\n    \n#     mean = np.mean(x_finite)\n#     std = np.std(x_finite)\n#     if std == 0:\n#         return x\n    \n#     x_standardized = (x - mean) / std\n#     x_standardized = np.clip(x_standardized, -5, 5)  # Limit range\n    \n#     from scipy.stats import skew\n#     skewness = skew(x_finite)\n#     delta = np.sign(skewness) * min(abs(skewness) / 3, 0.3)  # Reduced delta\n    \n#     x_transformed = x.copy()\n#     u = delta * x_standardized\n    \n#     mask_small = np.abs(u) < 0.01\n#     x_transformed[mask_small & finite_mask] = x_standardized[mask_small & finite_mask]\n    \n#     mask_large = ~mask_small & finite_mask\n#     if mask_large.any():\n#         u_large = u[mask_large & finite_mask]\n#         # Limit the argument to lambertw\n#         arg = np.clip(u_large * np.exp(u_large), -700, 700)\n        \n#         with np.errstate(invalid='ignore', over='ignore'):\n#             w_values = np.real(lambertw(arg))\n        \n#         # Replace any infinities\n#         w_values = handle_infinities(w_values, method='clip')\n        \n#         if abs(delta) > 1e-10:\n#             x_transformed[mask_large] = w_values / delta\n#         else:\n#             x_transformed[mask_large] = x_standardized[mask_large]\n    \n#     # Scale back\n#     x_transformed = x_transformed * std + mean\n    \n#     # Final clip\n#     x_transformed = clip_extreme_values(x_transformed)\n    \n#     return x_transformed\n\n# def apply_transformation(df, method, columns=None, **kwargs):\n#     \"\"\"Apply transformation to specified columns with robust infinity handling\"\"\"\n#     transformed_df = df.copy()\n    \n#     if columns is None:\n#         numeric_cols = df.select_dtypes(include=[np.number]).columns\n#         columns = [col for col in numeric_cols if col not in ['timestamp', 'label']]\n    \n#     for col in columns:\n#         original_values = df[col].values.copy()\n        \n#         # Pre-process: handle existing infinities\n#         original_values = handle_infinities(original_values, method='clip')\n        \n#         # Apply transformation\n#         if method == 'sigmoid':\n#             k = kwargs.get('k', 0.1)\n#             transformed_values = tapered_sigmoid_transform(original_values, k=k)\n            \n#         elif method == 'rank':\n#             rank_method = kwargs.get('rank_method', 'average')\n#             transformed_values = robust_rank_transform(original_values, method=rank_method)\n            \n#         elif method == 'robust_quantile':\n#             n_quantiles = kwargs.get('n_quantiles', 1000)\n#             subsample = kwargs.get('subsample', 100000)\n#             transformed_values = robust_quantile_transform(original_values, \n#                                                          n_quantiles=n_quantiles,\n#                                                          subsample=subsample)\n            \n#         elif method == 'sinh_arcsinh':\n#             epsilon = kwargs.get('epsilon', 0.1)\n#             delta = kwargs.get('delta', 1.0)\n#             transformed_values = sinh_arcsinh_transform(original_values, epsilon=epsilon, delta=delta)\n            \n#         elif method == 'yeo_johnson':\n#             try:\n#                 # Clip extreme values before transformation\n#                 clipped_values = clip_extreme_values(original_values, percentile_range=(0.1, 99.9))\n#                 pt = PowerTransformer(method='yeo-johnson', standardize=False)\n#                 transformed_values = pt.fit_transform(clipped_values.reshape(-1, 1)).ravel()\n#                 # Clip again after transformation\n#                 transformed_values = clip_extreme_values(transformed_values)\n#             except:\n#                 # Fallback to rank transform\n#                 transformed_values = robust_rank_transform(original_values)\n            \n#         elif method == 'quantile':\n#             try:\n#                 output_dist = kwargs.get('output_distribution', 'normal')\n#                 n_quantiles = kwargs.get('n_quantiles', 10000)\n                \n#                 # Use robust quantile transform instead\n#                 transformed_values = robust_quantile_transform(original_values, \n#                                                              n_quantiles=n_quantiles,\n#                                                              subsample=100000)\n                \n#                 if output_dist == 'normal':\n#                     finite_mask = np.isfinite(original_values)\n#                     if finite_mask.any():\n#                         scale = np.std(original_values[finite_mask])\n#                         center = np.median(original_values[finite_mask])\n#                         transformed_values = transformed_values * scale + center\n#                         transformed_values = clip_extreme_values(transformed_values)\n#             except:\n#                 # Fallback\n#                 transformed_values = robust_rank_transform(original_values)\n                \n#         elif method == 'novas':\n#             window_size = kwargs.get('window_size', 100)\n#             transformed_values = novas_transform(original_values, window_size=window_size)\n            \n#         elif method == 'directional_change':\n#             threshold = kwargs.get('threshold', 0.01)\n#             transformed_values = directional_change_transform(original_values, threshold=threshold)\n            \n#         elif method == 'modified_zscore':\n#             threshold = kwargs.get('threshold', 3.5)\n#             transformed_values = modified_zscore_transform(original_values, threshold=threshold)\n            \n#         elif method == 'asymmetric_winsorize':\n#             lower_pct = kwargs.get('lower_pct', 1)\n#             upper_pct = kwargs.get('upper_pct', 5)\n#             transformed_values = asymmetric_winsorize_transform(original_values, \n#                                                               lower_pct=lower_pct, \n#                                                               upper_pct=upper_pct)\n            \n#         elif method == 'log_robust':\n#             shift_method = kwargs.get('shift_method', 'auto')\n#             transformed_values = log_transform_robust(original_values, shift_method=shift_method)\n            \n#         elif method == 'huber':\n#             epsilon = kwargs.get('epsilon', 1.345)\n#             transformed_values = huber_transform(original_values, epsilon=epsilon)\n            \n#         elif method == 'box_cox':\n#             transformed_values = box_cox_robust(original_values)\n            \n#         elif method == 'lambert_w':\n#             transformed_values = lambert_w_transform(original_values)\n        \n#         else:\n#             raise ValueError(f\"Unknown transformation method: {method}\")\n        \n#         # Post-process: final check for infinities\n#         transformed_values = handle_infinities(transformed_values, method='clip')\n        \n#         # Ensure no extreme values remain\n#         if method not in ['rank', 'directional_change']:  # These are already bounded\n#             transformed_values = clip_extreme_values(transformed_values, percentile_range=(0.01, 99.99))\n        \n#         transformed_df[col] = transformed_values\n    \n#     return transformed_df\n\n# # =========================\n# # Training Functions\n# # =========================\n# def create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n#     positions = np.arange(n)\n#     normalized = positions / (n - 1)\n#     weights = decay ** (1.0 - normalized)\n#     return weights * n / weights.sum()\n\n# def get_model_slices(n_samples: int):\n#     return [\n#         {\"name\": \"full_data\", \"cutoff\": 0},\n#         {\"name\": \"last_75pct\", \"cutoff\": int(0.25 * n_samples)},\n#         {\"name\": \"last_50pct\", \"cutoff\": int(0.50 * n_samples)}\n#     ]\n\n# def train_and_evaluate(train_df, test_df):\n#     n_samples = len(train_df)\n#     model_slices = get_model_slices(n_samples)\n\n#     oof_preds = {\n#         learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n#         for learner in LEARNERS\n#     }\n#     test_preds = {\n#         learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n#         for learner in LEARNERS\n#     }\n\n#     full_weights = create_time_decay_weights(n_samples)\n#     kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n#     for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n#         print(f\"  Fold {fold}/{Config.N_FOLDS}\")\n#         X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n#         y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n#         for s in model_slices:\n#             cutoff = s[\"cutoff\"]\n#             slice_name = s[\"name\"]\n#             subset = train_df.iloc[cutoff:].reset_index(drop=True)\n#             rel_idx = train_idx[train_idx >= cutoff] - cutoff\n\n#             X_train = subset.iloc[rel_idx][Config.FEATURES]\n#             y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n#             sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n#             for learner in LEARNERS:\n#                 model = learner[\"Estimator\"](**learner[\"params\"])\n#                 model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n#                 mask = valid_idx >= cutoff\n#                 if mask.any():\n#                     idxs = valid_idx[mask]\n#                     oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n#                 if cutoff > 0 and (~mask).any():\n#                     oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n\n#                 test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[Config.FEATURES])\n\n#     # Normalize test predictions\n#     for learner_name in test_preds:\n#         for slice_name in test_preds[learner_name]:\n#             test_preds[learner_name][slice_name] /= Config.N_FOLDS\n\n#     return oof_preds, test_preds\n\n# def create_submission(train_df, oof_preds, test_preds, submission_df, method_name):\n#     learner_ensembles = {}\n#     for learner_name in oof_preds:\n#         scores = {s: pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[learner_name][s])[0]\n#                   for s in oof_preds[learner_name]}\n#         total_score = sum(scores.values())\n\n#         oof_simple = np.mean(list(oof_preds[learner_name].values()), axis=0)\n#         test_simple = np.mean(list(test_preds[learner_name].values()), axis=0)\n#         score_simple = pearsonr(train_df[Config.LABEL_COLUMN], oof_simple)[0]\n\n#         oof_weighted = sum(scores[s] / total_score * oof_preds[learner_name][s] for s in scores)\n#         test_weighted = sum(scores[s] / total_score * test_preds[learner_name][s] for s in scores)\n#         score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], oof_weighted)[0]\n\n#         print(f\"  {learner_name.upper()} Simple Ensemble Pearson:   {score_simple:.4f}\")\n#         print(f\"  {learner_name.upper()} Weighted Ensemble Pearson: {score_weighted:.4f}\")\n\n#         learner_ensembles[learner_name] = {\n#             \"oof_simple\": oof_simple,\n#             \"test_simple\": test_simple,\n#             \"oof_weighted\": oof_weighted,\n#             \"test_weighted\": test_weighted\n#         }\n\n#     final_oof = np.mean([le[\"oof_weighted\"] for le in learner_ensembles.values()], axis=0)\n#     final_test = np.mean([le[\"test_weighted\"] for le in learner_ensembles.values()], axis=0)\n#     final_score = pearsonr(train_df[Config.LABEL_COLUMN], final_oof)[0]\n\n#     print(f\"  FINAL ensemble Pearson: {final_score:.4f}\")\n\n#     submission_df[\"prediction\"] = final_test\n#     filename = f\"submission_{method_name}.csv\"\n#     submission_df.to_csv(filename, index=False)\n#     print(f\"  Saved: {filename}\")\n    \n#     return final_score\n\n# # =========================\n# # Main Execution\n# # =========================\n# def main():\n#     # Define all transformation methods with their parameters\n#     transformation_methods = [\n#         # Original methods (with safer parameters)\n#         {\"name\": \"robust_quantile\", \"params\": {\"n_quantiles\": 1000, \"subsample\": 100000}},\n#         {\"name\": \"rank\", \"params\": {\"rank_method\": \"average\"}},\n#         {\"name\": \"sigmoid\", \"params\": {\"k\": 0.05}},\n#         {\"name\": \"quantile\", \"params\": {\"output_distribution\": \"normal\", \"n_quantiles\": 10000}},\n#         {\"name\": \"sinh_arcsinh\", \"params\": {\"epsilon\": 0.1, \"delta\": 0.5}},  # Reduced delta\n#         {\"name\": \"yeo_johnson\", \"params\": {}},\n        \n#         # New advanced methods\n#         {\"name\": \"novas\", \"params\": {\"window_size\": 100}},\n#         {\"name\": \"directional_change\", \"params\": {\"threshold\": 0.01}},\n#         {\"name\": \"modified_zscore\", \"params\": {\"threshold\": 3.5}},\n#         {\"name\": \"asymmetric_winsorize\", \"params\": {\"lower_pct\": 1, \"upper_pct\": 5}},\n#         {\"name\": \"log_robust\", \"params\": {\"shift_method\": \"auto\"}},\n#         {\"name\": \"huber\", \"params\": {\"epsilon\": 1.345}},\n#         {\"name\": \"box_cox\", \"params\": {}},\n#         {\"name\": \"lambert_w\", \"params\": {}}\n#     ]\n    \n#     # Load raw data once\n#     print(\"Loading raw data...\")\n#     train_df_raw = pd.read_parquet(Config.TRAIN_PATH)\n#     test_df_raw = pd.read_parquet(Config.TEST_PATH)\n#     submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n#     print(f\"Raw data - Train: {train_df_raw.shape}, Test: {test_df_raw.shape}\")\n    \n#     # Check for infinities in raw data\n#     print(\"\\nChecking for infinities in raw data...\")\n#     for col in Config.FEATURES:\n#         n_inf_train = (~np.isfinite(train_df_raw[col])).sum()\n#         n_inf_test = (~np.isfinite(test_df_raw[col])).sum()\n#         if n_inf_train > 0 or n_inf_test > 0:\n#             print(f\"  {col}: Train infinities = {n_inf_train}, Test infinities = {n_inf_test}\")\n    \n#     # Reduce memory usage\n#     train_df_raw = reduce_mem_usage(train_df_raw, 'train', verbose=False)\n#     test_df_raw = reduce_mem_usage(test_df_raw, 'test', verbose=False)\n    \n#     results = []\n    \n#     # Process each transformation method\n#     for method_info in transformation_methods:\n#         method_name = method_info[\"name\"]\n#         method_params = method_info[\"params\"]\n        \n#         print(f\"\\n{'='*60}\")\n#         print(f\"Processing transformation method: {method_name.upper()}\")\n#         print(f\"{'='*60}\")\n        \n#         try:\n#             # Apply transformation only to FEATURES columns\n#             print(f\"Applying {method_name} transformation...\")\n#             train_df = train_df_raw.copy()\n#             test_df = test_df_raw.copy()\n            \n#             # Transform only the feature columns\n#             train_df[Config.FEATURES] = apply_transformation(\n#                 train_df[Config.FEATURES], \n#                 method_name, \n#                 columns=Config.FEATURES,\n#                 **method_params\n#             )[Config.FEATURES]\n            \n#             test_df[Config.FEATURES] = apply_transformation(\n#                 test_df[Config.FEATURES], \n#                 method_name, \n#                 columns=Config.FEATURES,\n#                 **method_params\n#             )[Config.FEATURES]\n            \n#             # Verify no infinities remain\n#             has_inf = False\n#             for col in Config.FEATURES:\n#                 if (~np.isfinite(train_df[col])).any() or (~np.isfinite(test_df[col])).any():\n#                     print(f\"  WARNING: Column {col} still has infinities after transformation!\")\n#                     has_inf = True\n            \n#             if has_inf:\n#                 print(f\"  Skipping {method_name} due to remaining infinities\")\n#                 continue\n            \n#             # Reset index\n#             train_df = train_df.reset_index(drop=True)\n#             test_df = test_df.reset_index(drop=True)\n            \n#             # Train and evaluate\n#             print(f\"Training models with {method_name} transformation...\")\n#             oof_preds, test_preds = train_and_evaluate(train_df, test_df)\n            \n#             # Create submission\n#             print(f\"Creating submission for {method_name}...\")\n#             final_score = create_submission(train_df, oof_preds, test_preds, \n#                                           submission_df.copy(), method_name)\n            \n#             results.append({\"method\": method_name, \"score\": final_score})\n            \n#         except Exception as e:\n#             print(f\"  ERROR with {method_name}: {str(e)}\")\n#             import traceback\n#             traceback.print_exc()\n#             continue\n    \n#     # Summary\n#     print(f\"\\n{'='*60}\")\n#     print(\"SUMMARY OF ALL METHODS\")\n#     print(f\"{'='*60}\")\n#     for result in sorted(results, key=lambda x: x['score'], reverse=True):\n#         print(f\"{result['method']:<25} Pearson: {result['score']:.4f}\")\n    \n#     if results:\n#         best_method = max(results, key=lambda x: x['score'])\n#         print(f\"\\nBest method: {best_method['method']} (Pearson: {best_method['score']:.4f})\")\n#         print(f\"Total submission files created: {len(results)}\")\n\n# if __name__ == \"__main__\":\n#     main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}