{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer\nfrom sklearn.feature_selection import SelectKBest, mutual_info_regression\nfrom sklearn.decomposition import PCA, TruncatedSVD\nfrom sklearn.cluster import KMeans\nfrom scipy.stats import pearsonr, rankdata, skew, kurtosis\nfrom scipy.optimize import minimize, differential_evolution\nimport gc\n\nprint(\"Starting Enhanced XGBoost Pipeline V2 - Maximum Performance...\")\nprint(\"=\" * 80)\n\n# Enhanced Configuration\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    n_folds = 5\n    random_state = 42\n    use_gpu = False\n    \n    # Feature settings - balanced but comprehensive\n    max_x_features = 100  # Increased from 80\n    n_interaction_features = 75  # Increased from 50\n    n_proprietary_features = 45  # Increased from 30\n    n_polynomial_features = 20  # New\n    n_ratio_features = 25  # New\n    \n    # Feature selection\n    use_feature_selection = True\n    feature_selection_threshold = 0.008  # Slightly lower threshold\n    max_final_features = 150  # Allow more features\n\n# Memory optimization\ndef reduce_mem_usage(df, name=\"\"):\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    return df\n\n# Enhanced proprietary features\ndef create_proprietary_x_variables(df, n_features=45):\n    \"\"\"Create comprehensive proprietary X variables\"\"\"\n    print(f\"Creating {n_features} proprietary X variables...\")\n    \n    # Get existing X features\n    x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n    \n    # Important X features from analysis\n    important_x = [\"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n                   \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n                   \"X271\", \"X272\", \"X273\", \"X274\", \"X275\"]\n    \n    # Use available important features\n    base_features = [f for f in important_x if f in df.columns][:15]\n    \n    # Add some high-variance features\n    if len(base_features) < 15:\n        x_variances = df[x_features].var()\n        high_var_features = x_variances.nlargest(15).index.tolist()\n        for feat in high_var_features:\n            if feat not in base_features:\n                base_features.append(feat)\n                if len(base_features) >= 15:\n                    break\n    \n    prop_idx = 1\n    \n    # 1. Enhanced statistical combinations (12 features)\n    if len(base_features) >= 10:\n        # Basic statistics\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].mean(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].std(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].max(axis=1) - df[base_features[:10]].min(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].median(axis=1)\n        prop_idx += 1\n        \n        # Percentiles\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].quantile(0.25, axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].quantile(0.75, axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].quantile(0.90, axis=1)\n        prop_idx += 1\n        \n        # Higher moments\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].skew(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].kurtosis(axis=1)\n        prop_idx += 1\n        \n        # Counts and positions\n        df[f'X_prop_{prop_idx}'] = (df[base_features[:10]] > df[base_features[:10]].mean(axis=1).values[:, None]).sum(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].idxmax(axis=1).str.extract('(\\d+)')[0].astype(float)\n        prop_idx += 1\n        \n        # Coefficient of variation\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].std(axis=1) / (df[base_features[:10]].mean(axis=1) + 1e-8)\n        prop_idx += 1\n    \n    # 2. Advanced non-linear transformations (10 features)\n    for i in range(10):\n        feat = base_features[i % len(base_features)]\n        if i < 3:\n            # Square root transformations\n            df[f'X_prop_{prop_idx}'] = np.sign(df[feat]) * np.sqrt(np.abs(df[feat]))\n        elif i < 6:\n            # Hyperbolic tangent\n            df[f'X_prop_{prop_idx}'] = np.tanh(df[feat] / df[feat].std())\n        elif i < 8:\n            # Sigmoid\n            df[f'X_prop_{prop_idx}'] = 1 / (1 + np.exp(-df[feat] / df[feat].std()))\n        else:\n            # Rank transform\n            df[f'X_prop_{prop_idx}'] = rankdata(df[feat]) / len(df)\n        prop_idx += 1\n    \n    # 3. Enhanced market interaction features (13 features)\n    market_features = ['volume', 'order_flow_imbalance', 'kyle_lambda', 'vpin', 'liquidity_imbalance']\n    \n    for i, mf in enumerate(market_features):\n        if mf in df.columns and prop_idx <= n_features - 5:\n            for j in range(min(3, n_features - prop_idx)):\n                bf = base_features[(i*3 + j) % len(base_features)]\n                if j == 0:\n                    df[f'X_prop_{prop_idx}'] = df[bf] * df[mf]\n                elif j == 1:\n                    df[f'X_prop_{prop_idx}'] = df[bf] * np.log1p(np.abs(df[mf]))\n                else:\n                    df[f'X_prop_{prop_idx}'] = df[bf] * np.sign(df[mf]) * np.sqrt(np.abs(df[mf]))\n                prop_idx += 1\n    \n    # 4. Clustering-based features (5 features)\n    if len(base_features) >= 10 and prop_idx <= n_features - 5:\n        # Cluster X features\n        X_subset = df[base_features[:10]].fillna(0)\n        \n        kmeans = KMeans(n_clusters=5, random_state=42, n_init=10)\n        clusters = kmeans.fit_predict(X_subset)\n        df[f'X_prop_{prop_idx}'] = clusters\n        prop_idx += 1\n        \n        # Distance to nearest cluster center\n        distances = kmeans.transform(X_subset)\n        df[f'X_prop_{prop_idx}'] = distances.min(axis=1)\n        prop_idx += 1\n        \n        # Cluster probability (soft assignment)\n        soft_clusters = np.exp(-distances) / np.exp(-distances).sum(axis=1, keepdims=True)\n        df[f'X_prop_{prop_idx}'] = soft_clusters.max(axis=1)\n        prop_idx += 1\n    \n    # 5. PCA features (3 features)\n    if len(base_features) >= 10 and prop_idx <= n_features - 3:\n        pca = PCA(n_components=3, random_state=42)\n        pca_features = pca.fit_transform(df[base_features[:10]].fillna(0))\n        for i in range(3):\n            df[f'X_prop_{prop_idx}'] = pca_features[:, i]\n            prop_idx += 1\n    \n    # 6. Interaction ratios (remaining features)\n    while prop_idx <= n_features:\n        i = prop_idx - 1\n        feat1 = base_features[i % len(base_features)]\n        feat2 = base_features[(i + 1) % len(base_features)]\n        df[f'X_prop_{prop_idx}'] = df[feat1] / (np.abs(df[feat2]) + 1e-8)\n        prop_idx += 1\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    print(f\"Created {prop_idx-1} proprietary X variables\")\n    return df\n\n# Create polynomial features\ndef create_polynomial_features(df, key_features, max_features=20):\n    \"\"\"Create polynomial combinations of key features\"\"\"\n    print(f\"Creating polynomial features...\")\n    \n    poly_idx = 1\n    \n    # Single feature polynomials\n    for feat in key_features[:8]:\n        if feat in df.columns and poly_idx <= max_features - 5:\n            df[f'poly_{poly_idx}_{feat}_2'] = df[feat] ** 2\n            poly_idx += 1\n            df[f'poly_{poly_idx}_{feat}_3'] = df[feat] ** 3\n            poly_idx += 1\n            df[f'poly_{poly_idx}_{feat}_sqrt'] = np.sqrt(np.abs(df[feat]))\n            poly_idx += 1\n    \n    # Cross-feature polynomials\n    for i in range(len(key_features)-1):\n        for j in range(i+1, min(i+3, len(key_features))):\n            if poly_idx > max_features:\n                break\n            if key_features[i] in df.columns and key_features[j] in df.columns:\n                df[f'poly_{poly_idx}_cross'] = df[key_features[i]] * df[key_features[j]]\n                poly_idx += 1\n    \n    return df\n\n# Create ratio features\ndef create_ratio_features(df):\n    \"\"\"Create comprehensive ratio features\"\"\"\n    print(\"Creating ratio features...\")\n    \n    ratio_pairs = [\n        # Volume ratios\n        ('buy_qty', 'volume'),\n        ('sell_qty', 'volume'),\n        ('bid_qty', 'ask_qty'),\n        ('buy_qty', 'sell_qty'),\n        ('bid_qty', 'total_liquidity'),\n        ('ask_qty', 'total_liquidity'),\n        \n        # Pressure ratios\n        ('buying_pressure', 'selling_pressure'),\n        ('bid_pressure', 'ask_pressure'),\n        \n        # Liquidity ratios\n        ('volume', 'total_liquidity'),\n        ('bid_ask_spread', 'total_liquidity'),\n        \n        # Advanced ratios\n        ('order_flow_imbalance', 'vpin'),\n        ('kyle_lambda', 'liquidity_imbalance'),\n        ('effective_spread', 'realized_spread'),\n    ]\n    \n    for num, den in ratio_pairs:\n        if num in df.columns and den in df.columns:\n            df[f'ratio_{num}_{den}'] = df[num] / (df[den] + 1e-8)\n            # Only add log ratio for some pairs to avoid too many features\n            if num in ['buy_qty', 'volume', 'kyle_lambda']:\n                df[f'ratio_log_{num}_{den}'] = np.log1p(np.abs(df[num])) / (np.log1p(np.abs(df[den])) + 1e-8)\n    \n    return df\n\n# Enhanced interaction features\ndef create_interaction_features(df, selected_features, n_interactions=75):\n    \"\"\"Create high-quality interaction features with unique names\"\"\"\n    print(f\"Creating {n_interactions} interaction features...\")\n    \n    interaction_features = []\n    feature_names = []\n    used_names = set()\n    \n    # Helper function to create unique feature name\n    def get_unique_name(base_name):\n        if base_name not in used_names:\n            used_names.add(base_name)\n            return base_name\n        else:\n            counter = 1\n            while f\"{base_name}_v{counter}\" in used_names:\n                counter += 1\n            unique_name = f\"{base_name}_v{counter}\"\n            used_names.add(unique_name)\n            return unique_name\n    \n    # Prioritize features\n    important_x = [\"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n                   \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\"]\n    \n    market_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'liquidity_imbalance',\n                      'bid_ask_spread', 'buying_pressure', 'selling_pressure', 'volume', \n                      'log_volume', 'sqrt_volume', 'total_liquidity', 'liquidity_ratio']\n    \n    proprietary_features = [f for f in selected_features if 'X_prop_' in f][:15]\n    \n    # Get available priority features\n    priority_x = [f for f in important_x if f in selected_features and f in df.columns][:12]\n    priority_market = [f for f in market_features if f in selected_features and f in df.columns][:10]\n    priority_prop = [f for f in proprietary_features if f in df.columns][:8]\n    \n    interaction_count = 0\n    \n    # 1. X features with market microstructure (25 interactions)\n    for i, x_feat in enumerate(priority_x[:10]):\n        if interaction_count >= 25:\n            break\n        for j, market_feat in enumerate(priority_market[:5]):\n            if interaction_count >= 25:\n                break\n            \n            # Multiplication\n            interaction_features.append(df[x_feat] * df[market_feat])\n            feature_names.append(get_unique_name(f'{x_feat}_x_{market_feat}'))\n            interaction_count += 1\n            \n            # Log interaction for key features\n            if interaction_count < 25 and market_feat in ['volume', 'kyle_lambda', 'total_liquidity']:\n                interaction_features.append(df[x_feat] * np.log1p(np.abs(df[market_feat])))\n                feature_names.append(get_unique_name(f'{x_feat}_x_log_{market_feat}'))\n                interaction_count += 1\n    \n    # 2. Market feature interactions (20 interactions)\n    market_triplets = [\n        ('order_flow_imbalance', 'kyle_lambda', 'vpin'),\n        ('buying_pressure', 'selling_pressure', 'volume'),\n        ('bid_ask_spread', 'total_liquidity', 'liquidity_imbalance'),\n        ('vpin', 'kyle_lambda', 'sqrt_volume'),\n    ]\n    \n    for feat1, feat2, feat3 in market_triplets:\n        if interaction_count >= 45:\n            break\n        if all(f in df.columns for f in [feat1, feat2, feat3]):\n            # Three-way product\n            interaction_features.append(df[feat1] * df[feat2] * df[feat3])\n            feature_names.append(get_unique_name(f'{feat1}_{feat2}_{feat3}'))\n            interaction_count += 1\n            \n            # Ratio combinations\n            interaction_features.append((df[feat1] * df[feat2]) / (df[feat3] + 1e-8))\n            feature_names.append(get_unique_name(f'{feat1}_{feat2}_div_{feat3}'))\n            interaction_count += 1\n            \n            # Sum and product\n            interaction_features.append((df[feat1] + df[feat2]) * df[feat3])\n            feature_names.append(get_unique_name(f'{feat1}_plus_{feat2}_x_{feat3}'))\n            interaction_count += 1\n            \n            # Difference and product\n            interaction_features.append((df[feat1] - df[feat2]) * df[feat3])\n            feature_names.append(get_unique_name(f'{feat1}_minus_{feat2}_x_{feat3}'))\n            interaction_count += 1\n            \n            # Max/min interactions\n            interaction_features.append(np.maximum(df[feat1], df[feat2]) * df[feat3])\n            feature_names.append(get_unique_name(f'max_{feat1}_{feat2}_x_{feat3}'))\n            interaction_count += 1\n    \n    # 3. Proprietary feature interactions (15 interactions)\n    for i, prop_feat in enumerate(priority_prop[:8]):\n        if interaction_count >= 60:\n            break\n        \n        # With market features\n        for j, market_feat in enumerate(priority_market[:3]):\n            if interaction_count >= 60:\n                break\n            \n            interaction_features.append(df[prop_feat] * df[market_feat])\n            feature_names.append(get_unique_name(f'{prop_feat}_x_{market_feat}'))\n            interaction_count += 1\n        \n        # With X features\n        if i < len(priority_x) and interaction_count < 60:\n            interaction_features.append(df[prop_feat] * df[priority_x[i]])\n            feature_names.append(get_unique_name(f'{prop_feat}_x_{priority_x[i]}'))\n            interaction_count += 1\n    \n    # 4. Non-linear transformations (10 interactions)\n    key_features = priority_x[:5] + priority_market[:5]\n    for i, feat in enumerate(key_features[:10]):\n        if interaction_count >= 70:\n            break\n        if feat in df.columns:\n            # Exponential decay\n            interaction_features.append(np.exp(-np.abs(df[feat]) / df[feat].std()))\n            feature_names.append(get_unique_name(f'{feat}_exp_decay'))\n            interaction_count += 1\n    \n    # 5. Statistical interactions (5 interactions)\n    if len(priority_x) >= 5:\n        # Mean of top X features\n        x_mean = df[priority_x[:5]].mean(axis=1)\n        x_std = df[priority_x[:5]].std(axis=1)\n        \n        # Interaction with market features\n        for mf in ['volume', 'order_flow_imbalance', 'kyle_lambda']:\n            if interaction_count >= n_interactions:\n                break\n            if mf in df.columns:\n                interaction_features.append(x_mean * df[mf])\n                feature_names.append(get_unique_name(f'x_mean_x_{mf}'))\n                interaction_count += 1\n                \n                if interaction_count < n_interactions:\n                    interaction_features.append(x_std * df[mf])\n                    feature_names.append(get_unique_name(f'x_std_x_{mf}'))\n                    interaction_count += 1\n    \n    # Create DataFrame\n    interaction_df = pd.DataFrame(\n        np.column_stack(interaction_features[:interaction_count]),\n        columns=feature_names[:interaction_count],\n        index=df.index\n    )\n    \n    # Handle infinities and NaN\n    interaction_df = interaction_df.replace([np.inf, -np.inf], np.nan)\n    interaction_df = interaction_df.fillna(0)\n    \n    print(f\"Created {interaction_count} interaction features\")\n    return interaction_df\n\n# Enhanced feature engineering\ndef add_features(df):\n    \"\"\"Create comprehensive features for market microstructure\"\"\"\n    print(\"Engineering features...\")\n    \n    eps = 1e-8\n    \n    # Basic interactions\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + eps)\n    df['ask_bid_ratio'] = df['ask_qty'] / (df['bid_qty'] + eps)\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + eps)\n    df['sell_buy_ratio'] = df['sell_qty'] / (df['buy_qty'] + eps)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    \n    # Pressure indicators\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + eps)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + eps)\n    df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n    df['pressure_ratio'] = df['buying_pressure'] / (df['selling_pressure'] + eps)\n    \n    # Liquidity features\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + eps)\n    df['liquidity_ratio'] = df['total_liquidity'] / (df['volume'] + eps)\n    df['liquidity_consumption'] = df['volume'] / (df['total_liquidity'] + eps)\n    \n    # Volume transformations\n    df['log_volume'] = np.log1p(df['volume'])\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['cbrt_volume'] = np.cbrt(df['volume'])\n    df['volume_squared'] = df['volume'] ** 2\n    \n    # Market microstructure\n    df['kyle_lambda'] = df['order_flow_imbalance'] / (df['sqrt_volume'] + eps)\n    df['kyle_lambda_squared'] = df['kyle_lambda'] ** 2\n    df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n    df['amihud_illiquidity'] = np.abs(df['order_flow_imbalance']) / (df['log_volume'] + eps)\n    \n    # Additional microstructure features\n    df['effective_spread'] = 2 * np.abs(df['order_flow_imbalance']) * df['bid_ask_spread']\n    df['realized_spread'] = df['bid_ask_spread'] * df['vpin']\n    df['price_impact'] = df['kyle_lambda'] * df['volume']\n    df['trade_intensity'] = df['volume'] / (df['total_liquidity'] + eps)\n    df['relative_spread'] = df['bid_ask_spread'] / (df['total_liquidity'] + eps)\n    \n    # Information asymmetry measures\n    df['pin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + eps)\n    df['adjusted_pin'] = df['pin'] * df['volume'] / (df['total_liquidity'] + eps)\n    \n    # Order book shape\n    df['book_imbalance_ratio'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + eps)\n    df['book_skew'] = (df['bid_qty'] - df['ask_qty']) / (df['volume'] + eps)\n    \n    # Trading activity\n    df['trade_size'] = df['volume'] / ((df['buy_qty'] + df['sell_qty']) / (df['bid_qty'] + df['ask_qty'] + eps) + eps)\n    df['order_book_depth'] = df['bid_qty'] + df['ask_qty']\n    df['relative_volume'] = df['volume'] / (df['order_book_depth'] + eps)\n    \n    # Volatility proxies\n    df['range_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + eps)\n    df['dispersion'] = np.abs(df['order_flow_imbalance']) * df['bid_ask_spread']\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    return df\n\n# Enhanced feature selection\ndef select_features_by_importance(X_train, y_train, feature_names, threshold=0.008, max_features=150):\n    \"\"\"Select features based on importance scores with enhanced logic\"\"\"\n    print(\"Calculating feature importance...\")\n    \n    # Train a quick model to get feature importance\n    params = {\n        'n_estimators': 150,\n        'max_depth': 6,\n        'learning_rate': 0.1,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'random_state': 42,\n        'n_jobs': -1,\n        'verbosity': 0\n    }\n    \n    model = XGBRegressor(**params)\n    model.fit(X_train, y_train)\n    \n    # Get feature importance\n    importance = model.feature_importances_\n    \n    # Create importance DataFrame\n    importance_df = pd.DataFrame({\n        'feature': feature_names,\n        'importance': importance\n    }).sort_values('importance', ascending=False)\n    \n    # Select features above threshold\n    selected_features = importance_df[importance_df['importance'] > threshold]['feature'].tolist()\n    \n    # Always include critical features\n    critical_features = [\n        'order_flow_imbalance', 'kyle_lambda', 'vpin', 'volume', 'log_volume',\n        'bid_ask_spread', 'liquidity_imbalance', 'buying_pressure', 'selling_pressure',\n        'total_liquidity', 'liquidity_ratio', 'amihud_illiquidity', 'pin',\n        'effective_spread', 'realized_spread', 'price_impact'\n    ]\n    \n    # Add critical proprietary features\n    critical_x = [\"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\"]\n    critical_features.extend([f for f in critical_x if f in feature_names])\n    \n    for feat in critical_features:\n        if feat in feature_names and feat not in selected_features:\n            selected_features.append(feat)\n    \n    # Limit to max_features\n    if len(selected_features) > max_features:\n        # Keep all critical features and top importance features\n        critical_in_selected = [f for f in critical_features if f in selected_features]\n        other_features = [f for f in selected_features if f not in critical_features]\n        \n        # Sort other features by importance\n        other_features_importance = importance_df[importance_df['feature'].isin(other_features)]\n        top_other = other_features_importance.head(max_features - len(critical_in_selected))['feature'].tolist()\n        \n        selected_features = critical_in_selected + top_other\n    \n    print(f\"Selected {len(selected_features)} features with importance > {threshold}\")\n    \n    return selected_features, importance_df\n\n# Enhanced Anti-overfitting strategies\nclass AntiOverfitXGB:\n    def __init__(self, base_params=None):\n        self.base_params = base_params or {\n            'tree_method': 'hist',\n            'n_estimators': 500,\n            'learning_rate': 0.01,\n            'max_depth': 6,\n            'random_state': 42,\n            'n_jobs': -1,\n            'verbosity': 0\n        }\n        \n    def train_overfit_model(self, X, y, overfit_direction='high'):\n        \"\"\"Train a model designed to overfit in a specific direction\"\"\"\n        params = self.base_params.copy()\n        \n        if overfit_direction == 'high':\n            params.update({\n                'max_depth': 12,\n                'min_child_weight': 1,\n                'subsample': 0.9,\n                'colsample_bytree': 0.9,\n                'reg_alpha': 0.001,\n                'reg_lambda': 0.001,\n                'learning_rate': 0.05,\n                'gamma': 0\n            })\n        elif overfit_direction == 'medium':\n            params.update({\n                'max_depth': 8,\n                'min_child_weight': 10,\n                'subsample': 0.7,\n                'colsample_bytree': 0.7,\n                'reg_alpha': 1,\n                'reg_lambda': 1,\n                'learning_rate': 0.02,\n                'gamma': 0.1\n            })\n        else:  # low\n            params.update({\n                'max_depth': 3,\n                'min_child_weight': 50,\n                'subsample': 0.5,\n                'colsample_bytree': 0.5,\n                'reg_alpha': 10,\n                'reg_lambda': 10,\n                'learning_rate': 0.001,\n                'gamma': 1\n            })\n        \n        model = XGBRegressor(**params)\n        model.fit(X, y)\n        return model\n    \n    def identify_overfit_samples(self, X, y, n_folds=5):\n        \"\"\"Identify samples where model tends to overfit\"\"\"\n        kf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n        \n        errors = np.zeros(len(X))\n        predictions = np.zeros(len(X))\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n            X_fold_train = X.iloc[train_idx] if hasattr(X, 'iloc') else X[train_idx]\n            y_fold_train = y.iloc[train_idx] if hasattr(y, 'iloc') else y[train_idx]\n            X_fold_valid = X.iloc[valid_idx] if hasattr(X, 'iloc') else X[valid_idx]\n            y_fold_valid = y.iloc[valid_idx] if hasattr(y, 'iloc') else y[valid_idx]\n            \n            model = self.train_overfit_model(X_fold_train, y_fold_train, 'high')\n            pred = model.predict(X_fold_valid)\n            predictions[valid_idx] = pred\n            errors[valid_idx] = np.abs(pred - y_fold_valid)\n        \n        # Multiple error thresholds\n        error_p75 = np.percentile(errors, 75)\n        error_p90 = np.percentile(errors, 90)\n        \n        overfit_mask = errors > error_p75\n        extreme_overfit_mask = errors > error_p90\n        \n        return overfit_mask, extreme_overfit_mask, predictions, errors\n    \n    def train_adversarial_ensemble(self, X, y, X_test):\n        \"\"\"Train enhanced ensemble with models that overfit in opposite directions\"\"\"\n        print(\"Training high-overfitting model...\")\n        model_high = self.train_overfit_model(X, y, 'high')\n        pred_high_train = model_high.predict(X)\n        pred_high_test = model_high.predict(X_test)\n        \n        print(\"Training medium-overfitting model...\")\n        model_medium = self.train_overfit_model(X, y, 'medium')\n        pred_medium_train = model_medium.predict(X)\n        pred_medium_test = model_medium.predict(X_test)\n        \n        print(\"Training low-overfitting model...\")\n        model_low = self.train_overfit_model(X, y, 'low')\n        pred_low_train = model_low.predict(X)\n        pred_low_test = model_low.predict(X_test)\n        \n        print(\"Training residual model...\")\n        # Weighted average of predictions\n        ensemble_train = (0.3 * pred_high_train + 0.4 * pred_medium_train + 0.3 * pred_low_train)\n        residuals = y - ensemble_train\n        \n        params = self.base_params.copy()\n        params.update({\n            'max_depth': 6,\n            'learning_rate': 0.02,\n            'subsample': 0.7,\n            'colsample_bytree': 0.7,\n            'reg_alpha': 2,\n            'reg_lambda': 2\n        })\n        model_residual = XGBRegressor(**params)\n        model_residual.fit(X, residuals)\n        pred_residual_train = model_residual.predict(X)\n        pred_residual_test = model_residual.predict(X_test)\n        \n        final_train = ensemble_train + pred_residual_train\n        final_test = (0.3 * pred_high_test + 0.4 * pred_medium_test + 0.3 * pred_low_test) + pred_residual_test\n        \n        return final_train, final_test\n    \n    def train_with_sample_weights(self, X, y, overfit_mask, extreme_overfit_mask):\n        \"\"\"Train model with graduated weights based on overfit tendency\"\"\"\n        sample_weights = np.ones(len(X))\n        sample_weights[overfit_mask] = 0.7\n        sample_weights[extreme_overfit_mask] = 0.3\n        \n        params = self.base_params.copy()\n        params.update({\n            'max_depth': 8,\n            'learning_rate': 0.02,\n            'subsample': 0.8,\n            'colsample_bytree': 0.8,\n            'reg_alpha': 1,\n            'reg_lambda': 1\n        })\n        \n        model = XGBRegressor(**params)\n        model.fit(X, y, sample_weight=sample_weights)\n        return model\n\n# Enhanced ensemble optimization\ndef optimize_ensemble_weights(predictions, y_true, method='advanced'):\n    \"\"\"Optimize ensemble weights using multiple advanced methods\"\"\"\n    n_models = len(predictions)\n    \n    def objective(weights):\n        weights = weights / weights.sum()\n        blended = np.sum([w * p for w, p in zip(weights, predictions)], axis=0)\n        return -pearsonr(y_true, blended)[0]\n    \n    def objective_with_penalty(weights):\n        weights = weights / weights.sum()\n        blended = np.sum([w * p for w, p in zip(weights, predictions)], axis=0)\n        corr = pearsonr(y_true, blended)[0]\n        # Add penalty for extreme weights\n        penalty = 0.1 * np.std(weights)\n        return -corr + penalty\n    \n    best_weights = None\n    best_score = float('inf')\n    \n    # Method 1: Equal weights\n    equal_weights = np.ones(n_models) / n_models\n    equal_score = objective(equal_weights)\n    if equal_score < best_score:\n        best_score = equal_score\n        best_weights = equal_weights\n    \n    # Method 2: SLSQP with constraints\n    constraints = {'type': 'eq', 'fun': lambda w: np.sum(w) - 1}\n    bounds = [(0, 1) for _ in range(n_models)]\n    \n    # Try multiple starting points\n    for _ in range(3):\n        x0 = np.random.dirichlet(np.ones(n_models))\n        result = minimize(objective_with_penalty, x0, method='SLSQP', \n                         bounds=bounds, constraints=constraints)\n        if result.success and result.fun < best_score:\n            best_score = result.fun\n            best_weights = result.x\n    \n    # Method 3: Differential Evolution\n    if method == 'advanced':\n        try:\n            result_de = differential_evolution(\n                objective_with_penalty, \n                bounds, \n                seed=42, \n                maxiter=200,\n                popsize=15,\n                tol=0.001\n            )\n            de_weights = result_de.x / result_de.x.sum()\n            if result_de.fun < best_score:\n                best_score = result_de.fun\n                best_weights = de_weights\n        except:\n            pass\n    \n    # Method 4: Performance-based weighting\n    individual_scores = [pearsonr(y_true, pred)[0] for pred in predictions]\n    performance_weights = np.array(individual_scores)\n    performance_weights = np.maximum(performance_weights, 0)  # Remove negative correlations\n    if performance_weights.sum() > 0:\n        performance_weights = performance_weights / performance_weights.sum()\n        perf_score = objective(performance_weights)\n        if perf_score < best_score:\n            best_score = perf_score\n            best_weights = performance_weights\n    \n    return best_weights\n\n# Main pipeline\nprint(\"\\nLoading data...\")\ntrain = pd.read_parquet(CFG.train_path)\ntest = pd.read_parquet(CFG.test_path)\nsubmission = pd.read_csv(CFG.sample_sub_path)\n\nprint(f\"Train shape: {train.shape}\")\nprint(f\"Test shape: {test.shape}\")\n\n# Create all types of features\nprint(\"\\n\" + \"=\"*60)\nprint(\"Feature Engineering Phase\")\nprint(\"=\"*60)\n\n# 1. Proprietary features\ntrain = create_proprietary_x_variables(train, CFG.n_proprietary_features)\ntest = create_proprietary_x_variables(test, CFG.n_proprietary_features)\n\n# 2. Market microstructure features\ntrain = add_features(train)\ntest = add_features(test)\n\n# 3. Ratio features\ntrain = create_ratio_features(train)\ntest = create_ratio_features(test)\n\n# Memory optimization\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\n# Select base features\nselected_x_features = [\n    \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n    \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\"\n]\n\n# Add proprietary features\nproprietary_features = [f\"X_prop_{i}\" for i in range(1, CFG.n_proprietary_features + 1)]\nselected_x_features.extend(proprietary_features)\n\n# Get all X features and add more if needed\nall_x_features = [col for col in train.columns if col.startswith('X') and col[1:].isdigit()]\nadditional_x = [f for f in all_x_features if f not in selected_x_features][:CFG.max_x_features - len(selected_x_features)]\nselected_x_features.extend(additional_x)\n\navailable_x_features = [f for f in selected_x_features if f in train.columns]\n\n# Market and engineered features\nmarket_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\nengineered_features = [col for col in train.columns if col not in market_features and \n                      col not in available_x_features and col != 'label' and col != 'timestamp']\n\n# Combine base features\nbase_selected_features = market_features + available_x_features + engineered_features\nbase_selected_features = list(dict.fromkeys(base_selected_features))\nbase_selected_features = [f for f in base_selected_features if f in train.columns]\n\nprint(f\"\\nBase features: {len(base_selected_features)}\")\n\n# 4. Polynomial features\nkey_features_for_poly = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'liquidity_imbalance', \n                        'X752', 'X287', 'X298', 'volume', 'bid_ask_spread']\ntrain = create_polynomial_features(train, key_features_for_poly, CFG.n_polynomial_features)\ntest = create_polynomial_features(test, key_features_for_poly, CFG.n_polynomial_features)\n\n# Update base features to include polynomial features\npoly_features = [col for col in train.columns if col.startswith('poly_')]\nbase_selected_features.extend(poly_features)\n\n# 5. Create interaction features\ninteraction_df_train = create_interaction_features(train, base_selected_features, CFG.n_interaction_features)\ninteraction_df_test = create_interaction_features(test, base_selected_features, CFG.n_interaction_features)\n\n# Add interaction features using concat\ntrain = pd.concat([train, interaction_df_train], axis=1)\ntest = pd.concat([test, interaction_df_test], axis=1)\n\n# All features before selection\nall_features = base_selected_features + list(interaction_df_train.columns)\nall_features = [f for f in all_features if f in train.columns]\nprint(f\"Total features before selection: {len(all_features)}\")\n\n# Prepare data for feature selection\nX_train_all = train[all_features]\ny_train = train['label']\nX_test_all = test[all_features]\n\n# Feature selection\nif CFG.use_feature_selection:\n    selected_features, importance_df = select_features_by_importance(\n        X_train_all, y_train, all_features, CFG.feature_selection_threshold, CFG.max_final_features\n    )\n    print(f\"\\nTop 15 features by importance:\")\n    print(importance_df.head(15))\nelse:\n    selected_features = all_features\n\nX_train = X_train_all[selected_features]\nX_test = X_test_all[selected_features]\n\nprint(f\"\\nFinal features after selection: {len(selected_features)}\")\n\n# Feature composition analysis\nx_features_selected = [f for f in selected_features if f.startswith('X') and not f.startswith('X_')]\nprop_features_selected = [f for f in selected_features if f.startswith('X_prop_')]\nmarket_features_selected = [f for f in selected_features if f in market_features]\nengineered_features_selected = [f for f in selected_features if f not in x_features_selected and \n                               f not in prop_features_selected and f not in market_features_selected]\n\nprint(f\"\\nFeature composition:\")\nprint(f\"  Original X features: {len(x_features_selected)}\")\nprint(f\"  Proprietary features: {len(prop_features_selected)}\")\nprint(f\"  Market features: {len(market_features_selected)}\")\nprint(f\"  Engineered features: {len(engineered_features_selected)}\")\n\n# Initialize storage for predictions\nall_predictions_train = []\nall_predictions_test = []\nall_scores = []\nmodel_names = []\n\n# Initialize anti-overfitting trainer\nanti_overfit = AntiOverfitXGB()\n\n# Strategy 1: Standard scaling with adversarial ensemble\nprint(\"\\n\" + \"=\"*60)\nprint(\"Strategy 1: Enhanced Adversarial Ensemble (Standard Scaling)\")\nprint(\"=\"*60)\n\nscaler = StandardScaler()\nX_train_scaled = pd.DataFrame(\n    scaler.fit_transform(X_train),\n    columns=selected_features,\n    index=X_train.index\n)\nX_test_scaled = pd.DataFrame(\n    scaler.transform(X_test),\n    columns=selected_features,\n    index=X_test.index\n)\n\n# Identify overfit-prone samples\nprint(\"\\nIdentifying overfit-prone samples...\")\noverfit_mask, extreme_overfit_mask, oof_predictions, errors = anti_overfit.identify_overfit_samples(\n    X_train_scaled, y_train, n_folds=5\n)\nprint(f\"Found {overfit_mask.sum()} overfit-prone samples ({100*overfit_mask.mean():.1f}%)\")\nprint(f\"Found {extreme_overfit_mask.sum()} extreme overfit samples ({100*extreme_overfit_mask.mean():.1f}%)\")\n\n# Train adversarial ensemble\nprint(\"\\nTraining enhanced adversarial ensemble...\")\nadv_train, adv_test = anti_overfit.train_adversarial_ensemble(\n    X_train_scaled, y_train, X_test_scaled\n)\n\nadv_score = pearsonr(y_train, adv_train)[0]\nprint(f\"Adversarial ensemble score: {adv_score:.4f}\")\n\nall_predictions_train.append(adv_train)\nall_predictions_test.append(adv_test)\nall_scores.append(adv_score)\nmodel_names.append('adversarial_standard')\n\n# Train with graduated sample weights\nprint(\"\\nTraining with graduated sample weights...\")\nweighted_model = anti_overfit.train_with_sample_weights(\n    X_train_scaled, y_train, overfit_mask, extreme_overfit_mask\n)\nweighted_train = weighted_model.predict(X_train_scaled)\nweighted_test = weighted_model.predict(X_test_scaled)\n\nweighted_score = pearsonr(y_train, weighted_train)[0]\nprint(f\"Weighted model score: {weighted_score:.4f}\")\n\nall_predictions_train.append(weighted_train)\nall_predictions_test.append(weighted_test)\nall_scores.append(weighted_score)\nmodel_names.append('weighted_standard')\n\n# Strategy 2: Multiple scalers with diverse models\nprint(\"\\n\" + \"=\"*60)\nprint(\"Strategy 2: Multi-Scaler Diverse Models\")\nprint(\"=\"*60)\n\nscalers = {\n    'robust': RobustScaler(),\n    'quantile': QuantileTransformer(n_quantiles=1000, output_distribution='normal', random_state=42)\n}\n\nfor scaler_name, scaler_obj in scalers.items():\n    print(f\"\\nProcessing with {scaler_name} scaler...\")\n    \n    X_train_transformed = pd.DataFrame(\n        scaler_obj.fit_transform(X_train),\n        columns=selected_features,\n        index=X_train.index\n    )\n    X_test_transformed = pd.DataFrame(\n        scaler_obj.transform(X_test),\n        columns=selected_features,\n        index=X_test.index\n    )\n    \n    # Train diverse models\n    diverse_configs = [\n        {\n            'name': f'conservative_{scaler_name}',\n            'params': {\n                'n_estimators': 1000,\n                'max_depth': 5,\n                'learning_rate': 0.01,\n                'subsample': 0.6,\n                'colsample_bytree': 0.6,\n                'reg_alpha': 5,\n                'reg_lambda': 5,\n                'min_child_weight': 30,\n                'gamma': 0.5\n            }\n        },\n        {\n            'name': f'balanced_{scaler_name}',\n            'params': {\n                'n_estimators': 800,\n                'max_depth': 7,\n                'learning_rate': 0.015,\n                'subsample': 0.75,\n                'colsample_bytree': 0.75,\n                'reg_alpha': 2,\n                'reg_lambda': 2,\n                'min_child_weight': 15,\n                'gamma': 0.2\n            }\n        },\n        {\n            'name': f'aggressive_{scaler_name}',\n            'params': {\n                'n_estimators': 600,\n                'max_depth': 9,\n                'learning_rate': 0.02,\n                'subsample': 0.85,\n                'colsample_bytree': 0.85,\n                'reg_alpha': 0.5,\n                'reg_lambda': 0.5,\n                'min_child_weight': 5,\n                'gamma': 0.1\n            }\n        }\n    ]\n    \n    for config in diverse_configs:\n        print(f\"  Training {config['name']} model...\")\n        params = anti_overfit.base_params.copy()\n        params.update(config['params'])\n        \n        model = XGBRegressor(**params)\n        model.fit(X_train_transformed, y_train)\n        \n        pred_train = model.predict(X_train_transformed)\n        pred_test = model.predict(X_test_transformed)\n        score = pearsonr(y_train, pred_train)[0]\n        \n        print(f\"    Score: {score:.4f}\")\n        \n        all_predictions_train.append(pred_train)\n        all_predictions_test.append(pred_test)\n        all_scores.append(score)\n        model_names.append(config['name'])\n\n# Strategy 3: Time windows (original approach)\nprint(\"\\n\" + \"=\"*60)\nprint(\"Strategy 3: Time-Window Models\")\nprint(\"=\"*60)\n\nwindows = [\n    {'name': 'recent_80', 'start': int(0.8 * len(train)), 'end': len(train)},\n    {'name': 'recent_60', 'start': int(0.6 * len(train)), 'end': len(train)},\n    {'name': 'recent_40', 'start': int(0.4 * len(train)), 'end': len(train)},\n    {'name': 'middle', 'start': int(0.2 * len(train)), 'end': int(0.8 * len(train))}\n]\n\nfor window in windows:\n    print(f\"\\nTraining on {window['name']} window...\")\n    X_window = X_train_scaled.iloc[window['start']:window['end']]\n    y_window = y_train.iloc[window['start']:window['end']]\n    \n    params = anti_overfit.base_params.copy()\n    params.update({\n        'max_depth': 7,\n        'learning_rate': 0.018,\n        'subsample': 0.75,\n        'colsample_bytree': 0.75,\n        'n_estimators': 800,\n        'reg_alpha': 1.5,\n        'reg_lambda': 1.5\n    })\n    \n    model = XGBRegressor(**params)\n    model.fit(X_window, y_window)\n    \n    pred_test = model.predict(X_test_scaled)\n    \n    # For scoring, predict on the window\n    pred_window = model.predict(X_window)\n    window_score = pearsonr(y_window, pred_window)[0]\n    \n    print(f\"  Window score: {window_score:.4f}\")\n    \n    all_predictions_test.append(pred_test)\n    all_predictions_train.append(np.zeros_like(y_train))  # Placeholder\n    all_scores.append(window_score)\n    model_names.append(f'window_{window[\"name\"]}')\n\n# Print model summary\nprint(\"\\n\" + \"=\"*60)\nprint(\"Model Performance Summary\")\nprint(\"=\"*60)\nfor name, score in zip(model_names, all_scores):\n    print(f\"{name}: {score:.4f}\")\n\n# Advanced ensemble optimization\nprint(\"\\n\" + \"=\"*60)\nprint(\"Creating Advanced Optimized Ensemble\")\nprint(\"=\"*60)\n\n# Use models with valid scores for optimization\nvalid_indices = [i for i, score in enumerate(all_scores) if score > 0.5]\nvalid_predictions_train = [all_predictions_train[i] for i in valid_indices]\nvalid_model_names = [model_names[i] for i in valid_indices]\n\n# Optimize weights\nprint(\"\\nOptimizing ensemble weights with advanced methods...\")\noptimal_weights = optimize_ensemble_weights(valid_predictions_train, y_train, method='advanced')\n\nprint(\"\\nOptimal weights:\")\nweight_summary = []\nfor name, weight in zip(valid_model_names, optimal_weights):\n    if weight > 0.01:\n        weight_summary.append((name, weight))\nweight_summary.sort(key=lambda x: x[1], reverse=True)\n\nfor name, weight in weight_summary:\n    print(f\"  {name}: {weight:.3f}\")\n\n# Create final ensemble\nfinal_predictions = np.zeros_like(all_predictions_test[0])\n\n# Apply optimized weights to valid models\nfor i, idx in enumerate(valid_indices):\n    final_predictions += optimal_weights[i] * all_predictions_test[idx]\n\n# Add window predictions with adaptive weight\nwindow_indices = [i for i, name in enumerate(model_names) if 'window' in name]\nif window_indices:\n    window_predictions = [all_predictions_test[i] for i in window_indices]\n    window_scores = [all_scores[i] for i in window_indices]\n    \n    # Weight windows by their scores\n    window_weights = np.array(window_scores)\n    window_weights = window_weights / window_weights.sum()\n    \n    window_avg = np.sum([w * p for w, p in zip(window_weights, window_predictions)], axis=0)\n    \n    # Adaptive blending weight based on model performance\n    ensemble_weight = 0.65 if max(all_scores) > 0.95 else 0.75\n    window_weight = 1 - ensemble_weight\n    \n    final_predictions = ensemble_weight * final_predictions + window_weight * window_avg\n\n# Enhanced post-processing\nprint(\"\\nApplying enhanced post-processing...\")\n\n# 1. Distribution matching\np1, p5, p95, p99 = np.percentile(y_train, [1, 5, 95, 99])\nmean_train, std_train = y_train.mean(), y_train.std()\n\n# Soft clipping with graduated adjustment\nextreme_high = final_predictions > p99\nhigh = (final_predictions > p95) & (~extreme_high)\nextreme_low = final_predictions < p1\nlow = (final_predictions < p5) & (~extreme_low)\n\nfinal_predictions[extreme_high] = p99 + 0.2 * (final_predictions[extreme_high] - p99)\nfinal_predictions[high] = p95 + 0.5 * (final_predictions[high] - p95)\nfinal_predictions[extreme_low] = p1 + 0.2 * (final_predictions[extreme_low] - p1)\nfinal_predictions[low] = p5 + 0.5 * (final_predictions[low] - p5)\n\n# 2. Ensure reasonable distribution\nfinal_predictions = np.clip(final_predictions, p1 - 0.5 * std_train, p99 + 0.5 * std_train)\n\n# 3. Mild calibration\npred_mean, pred_std = final_predictions.mean(), final_predictions.std()\nif pred_std > 0:\n    calibration_factor = np.clip(std_train / pred_std, 0.85, 1.15)\n    final_predictions = pred_mean + (final_predictions - pred_mean) * calibration_factor\n    final_predictions = final_predictions - final_predictions.mean() + mean_train\n\n# Create submission\nsubmission['prediction'] = final_predictions\nsubmission.to_csv('submission_enhanced_v2.csv', index=False)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"Enhanced submission saved to submission_enhanced_v2.csv\")\nprint(submission.head(10))\n\n# Save detailed predictions and analysis\npredictions_df = pd.DataFrame({\n    'adversarial': all_predictions_test[0],\n    'weighted': all_predictions_test[1],\n    'final': final_predictions\n})\npredictions_df.to_csv('enhanced_prediction_components_v2.csv', index=False)\n\nprint(\"\\nPrediction statistics:\")\nprint(predictions_df.describe())\n\n# Feature importance summary\nprint(\"\\n\" + \"=\"*80)\nprint(\"Feature Importance Summary\")\nprint(\"=\"*80)\nprint(\"\\nTop 20 most important features:\")\nprint(importance_df.head(20)[['feature', 'importance']])\n\n# Model diversity analysis\nprint(\"\\n\" + \"=\"*80)\nprint(\"Model Diversity Analysis\")\nprint(\"=\"*80)\ncorrelations = np.corrcoef(valid_predictions_train)\nmean_correlation = np.mean(correlations[np.triu_indices_from(correlations, k=1)])\nprint(f\"Average correlation between models: {mean_correlation:.3f}\")\nprint(f\"Model diversity score: {1 - mean_correlation:.3f}\")\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"Enhanced pipeline V2 completed successfully!\")\nprint(f\"Total features engineered: {len(all_features)}\")\nprint(f\"Features selected: {len(selected_features)}\")\nprint(f\"Models trained: {len(model_names)}\")\nprint(f\"Effective models in ensemble: {len(valid_indices)}\")\nprint(\"=\"*80)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}