{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom xgboost import XGBRegressor\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.stats import pearsonr\nfrom scipy.optimize import minimize\nimport gc\n\nprint(\"Starting XGBoost Anti-Overfitting Pipeline with Extended Features...\")\nprint(\"=\" * 80)\n\n# Configuration\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    n_folds = 5\n    random_state = 42\n    use_gpu = False  # Disabled due to GPU issues\n    max_x_features = 150  # Increased to include more X features\n    n_interaction_features = 100  # Doubled interaction features\n    n_proprietary_features = 50  # Increased from 30\n\n# Memory optimization\ndef reduce_mem_usage(df, name=\"\"):\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    return df\n\n# Function to create extensive feature interactions\ndef create_feature_interactions(df, selected_features, n_interactions=100):\n    \"\"\"Create comprehensive feature interactions\"\"\"\n    print(f\"Creating {n_interactions} feature interaction variables...\")\n    \n    interaction_features = []\n    feature_names = []\n    \n    # Get the most important features for interactions\n    important_x = [\"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n                   \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n                   \"X456\", \"X789\", \"X234\", \"X567\", \"X890\", \"X123\", \"X347\", \"X459\", \"X678\", \"X901\"]\n    \n    market_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume',\n                      'order_flow_imbalance', 'kyle_lambda', 'vpin', 'liquidity_imbalance',\n                      'bid_ask_spread', 'bid_ask_ratio', 'buy_sell_ratio', 'buying_pressure',\n                      'selling_pressure', 'total_liquidity', 'log_volume', 'sqrt_volume']\n    \n    # Select features available in the dataframe\n    priority_features = []\n    for feat in important_x + market_features:\n        if feat in selected_features and feat in df.columns:\n            priority_features.append(feat)\n    \n    # Add proprietary features to priority list\n    prop_features = [f for f in selected_features if 'X_prop_' in f][:20]\n    priority_features.extend(prop_features)\n    \n    # Add some random X features if we need more\n    other_x_features = [f for f in selected_features if f.startswith('X') and f not in priority_features]\n    priority_features.extend(other_x_features[:30])\n    \n    # Limit to available features\n    priority_features = priority_features[:50]\n    \n    interaction_count = 0\n    \n    # 1. Two-way interactions between top features (30 interactions)\n    for i in range(min(15, len(priority_features))):\n        for j in range(i+1, min(i+3, len(priority_features))):\n            if interaction_count >= 30:\n                break\n            feat1, feat2 = priority_features[i], priority_features[j]\n            \n            # Multiplication\n            interaction_features.append(df[feat1] * df[feat2])\n            feature_names.append(f'{feat1}_x_{feat2}')\n            interaction_count += 1\n            \n            # Division (protected)\n            if interaction_count < 30:\n                interaction_features.append(df[feat1] / (np.abs(df[feat2]) + 1e-8))\n                feature_names.append(f'{feat1}_div_{feat2}')\n                interaction_count += 1\n    \n    # 2. X features with market microstructure interactions (25 interactions)\n    x_features_for_market = [f for f in priority_features if f.startswith('X')][:15]\n    market_features_for_x = [f for f in priority_features if not f.startswith('X')][:10]\n    \n    for i, x_feat in enumerate(x_features_for_market):\n        if interaction_count >= 55:\n            break\n        for j, market_feat in enumerate(market_features_for_x[:3]):\n            if interaction_count >= 55:\n                break\n            \n            # Create meaningful market microstructure interactions\n            interaction_features.append(df[x_feat] * df[market_feat])\n            feature_names.append(f'{x_feat}_x_{market_feat}')\n            interaction_count += 1\n            \n            # Add log-transformed interaction\n            if interaction_count < 55:\n                interaction_features.append(df[x_feat] * np.log1p(np.abs(df[market_feat])))\n                feature_names.append(f'{x_feat}_x_log_{market_feat}')\n                interaction_count += 1\n    \n    # 3. Three-way interactions for most important features (20 interactions)\n    if len(priority_features) >= 3:\n        for i in range(min(10, len(priority_features)-2)):\n            if interaction_count >= 75:\n                break\n            feat1, feat2, feat3 = priority_features[i], priority_features[i+1], priority_features[i+2]\n            \n            # Three-way multiplication\n            interaction_features.append(df[feat1] * df[feat2] * df[feat3])\n            feature_names.append(f'{feat1}_x_{feat2}_x_{feat3}')\n            interaction_count += 1\n            \n            # Ratio of product\n            if interaction_count < 75:\n                interaction_features.append((df[feat1] * df[feat2]) / (np.abs(df[feat3]) + 1e-8))\n                feature_names.append(f'{feat1}_{feat2}_div_{feat3}')\n                interaction_count += 1\n    \n    # 4. Non-linear interactions (15 interactions)\n    for i in range(min(15, len(priority_features)-1)):\n        if interaction_count >= 90:\n            break\n        feat1, feat2 = priority_features[i], priority_features[i+1]\n        \n        # Squared difference\n        interaction_features.append((df[feat1] - df[feat2]) ** 2)\n        feature_names.append(f'{feat1}_minus_{feat2}_sq')\n        interaction_count += 1\n        \n        # Absolute difference\n        if interaction_count < 90:\n            interaction_features.append(np.abs(df[feat1] - df[feat2]))\n            feature_names.append(f'{feat1}_minus_{feat2}_abs')\n            interaction_count += 1\n    \n    # 5. Advanced polynomial interactions (10 interactions)\n    for i in range(min(5, len(priority_features))):\n        if interaction_count >= n_interactions:\n            break\n        feat = priority_features[i]\n        \n        # Cubic interaction\n        interaction_features.append(df[feat] ** 3)\n        feature_names.append(f'{feat}_cubed')\n        interaction_count += 1\n        \n        # Square root of absolute value\n        if interaction_count < n_interactions:\n            interaction_features.append(np.sqrt(np.abs(df[feat])))\n            feature_names.append(f'{feat}_sqrt_abs')\n            interaction_count += 1\n    \n    # Create DataFrame with interaction features\n    interaction_df = pd.DataFrame(\n        np.column_stack(interaction_features),\n        columns=feature_names[:len(interaction_features)],\n        index=df.index\n    )\n    \n    # Handle infinities and NaN\n    interaction_df = interaction_df.replace([np.inf, -np.inf], np.nan)\n    interaction_df = interaction_df.fillna(0)\n    \n    print(f\"Created {len(interaction_features)} interaction features\")\n    \n    return interaction_df\n\n# Function to create 50 new proprietary X variables\ndef create_proprietary_x_variables(df):\n    \"\"\"Create 50 new proprietary X variables based on existing features and domain knowledge\"\"\"\n    print(\"Creating 50 new proprietary X variables...\")\n    \n    # Get existing X features\n    x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n    print(f\"Found {len(x_features)} existing X features\")\n    \n    # Select the most important X features based on the provided list\n    important_x = [\"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n                   \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\"]\n    \n    # Use available important features\n    base_features = [f for f in important_x if f in df.columns][:15]\n    \n    # If we don't have enough important features, add some others\n    if len(base_features) < 15:\n        other_x = [f for f in x_features if f not in base_features]\n        base_features.extend(other_x[:15-len(base_features)])\n    \n    # Create 50 new variables with meaningful transformations\n    prop_idx = 1\n    \n    # 1. Non-linear transformations (10 variables)\n    for i in range(10):\n        feat = base_features[i % len(base_features)]\n        if i < 3:\n            df[f'X_prop_{prop_idx}'] = np.sign(df[feat]) * np.sqrt(np.abs(df[feat]))\n        elif i < 6:\n            df[f'X_prop_{prop_idx}'] = np.tanh(df[feat])\n        elif i < 8:\n            df[f'X_prop_{prop_idx}'] = np.exp(-np.abs(df[feat]))\n        else:\n            df[f'X_prop_{prop_idx}'] = 1 / (1 + np.exp(-df[feat]))  # Sigmoid\n        prop_idx += 1\n    \n    # 2. Interaction features between important X variables (10 variables)\n    for i in range(5):\n        for j in range(i+1, min(i+3, len(base_features))):\n            if prop_idx > 20:\n                break\n            feat1 = base_features[i]\n            feat2 = base_features[j]\n            df[f'X_prop_{prop_idx}'] = df[feat1] * df[feat2]\n            prop_idx += 1\n    \n    # 3. Ratio features (5 variables)\n    for i in range(5):\n        feat1 = base_features[i % len(base_features)]\n        feat2 = base_features[(i + 1) % len(base_features)]\n        df[f'X_prop_{prop_idx}'] = df[feat1] / (np.abs(df[feat2]) + 1e-8)\n        prop_idx += 1\n    \n    # 4. Statistical combinations (10 variables)\n    if len(base_features) >= 5:\n        # Various statistical measures\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].mean(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].std(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].max(axis=1) - df[base_features[:5]].min(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].median(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].skew(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].kurt(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:7]].quantile(0.25, axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:7]].quantile(0.75, axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].sum(axis=1)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:5]].prod(axis=1)\n        prop_idx += 1\n    \n    # 5. Market-based proprietary features (8 variables)\n    if 'volume' in df.columns:\n        for i in range(4):\n            df[f'X_prop_{prop_idx}'] = df[base_features[i]] * np.log1p(df['volume'])\n            prop_idx += 1\n    if 'bid_qty' in df.columns and 'ask_qty' in df.columns:\n        for i in range(2):\n            df[f'X_prop_{prop_idx}'] = df[base_features[i+4]] * (df['bid_qty'] - df['ask_qty'])\n            prop_idx += 1\n    if 'buy_qty' in df.columns and 'sell_qty' in df.columns:\n        df[f'X_prop_{prop_idx}'] = df[base_features[6]] * (df['buy_qty'] / (df['sell_qty'] + 1e-8))\n        prop_idx += 1\n    if 'volume' in df.columns:\n        df[f'X_prop_{prop_idx}'] = df[base_features[7]] / (np.sqrt(df['volume']) + 1e-8)\n        prop_idx += 1\n    \n    # 6. Polynomial and trigonometric features (7 variables)\n    for i in range(7):\n        feat = base_features[i % len(base_features)]\n        if i < 3:\n            df[f'X_prop_{prop_idx}'] = df[feat] ** 2\n        elif i < 5:\n            df[f'X_prop_{prop_idx}'] = df[feat] ** 3\n        elif i == 5:\n            df[f'X_prop_{prop_idx}'] = np.sin(df[feat])\n        else:\n            df[f'X_prop_{prop_idx}'] = np.cos(df[feat])\n        prop_idx += 1\n    \n    # Handle any infinities or NaN values created\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    print(f\"Created 50 new proprietary X variables (X_prop_1 to X_prop_50)\")\n    return df\n\n# Extended feature engineering\ndef add_features(df):\n    \"\"\"Create comprehensive features for market microstructure\"\"\"\n    print(\"Engineering extended features...\")\n    \n    # Basic interactions\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-8)\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    # Pressure indicators\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n    \n    # Liquidity features\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)\n    df['liquidity_concentration'] = df['total_liquidity'] / (df['volume'] + 1e-8)\n    \n    # Volume transformations\n    df['log_volume'] = np.log1p(df['volume'])\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['volume_squared'] = df['volume'] ** 2\n    df['inv_volume'] = 1 / (df['volume'] + 1e-8)\n    \n    # Market microstructure\n    df['kyle_lambda'] = df['order_flow_imbalance'] / (df['sqrt_volume'] + 1e-8)\n    df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['roll_measure'] = 2 * np.sqrt(np.abs(df['bid_ask_spread'] * df['order_flow_imbalance']))\n    \n    # Advanced microstructure\n    df['effective_spread'] = 2 * np.abs(df['order_flow_imbalance']) * df['bid_ask_spread']\n    df['realized_spread'] = df['bid_ask_spread'] * df['vpin']\n    df['price_impact'] = df['kyle_lambda'] * df['volume']\n    \n    # Orderbook features\n    df['orderbook_slope'] = (df['ask_qty'] - df['bid_qty']) / (df['bid_ask_spread'] + 1e-8)\n    df['mid_quote_change'] = df['bid_ask_spread'] * df['order_flow_imbalance']\n    df['weighted_mid_price'] = (df['bid_qty'] * df['ask_qty']) / (df['total_liquidity'] + 1e-8)\n    \n    # Trading intensity\n    df['trade_intensity'] = df['volume'] / (df['total_liquidity'] + 1e-8)\n    df['buy_intensity'] = df['buy_qty'] / (df['bid_qty'] + 1e-8)\n    df['sell_intensity'] = df['sell_qty'] / (df['ask_qty'] + 1e-8)\n    \n    # Normalized features\n    df['norm_bid_qty'] = df['bid_qty'] / (df['volume'] + 1e-8)\n    df['norm_ask_qty'] = df['ask_qty'] / (df['volume'] + 1e-8)\n    df['norm_buy_qty'] = df['buy_qty'] / (df['volume'] + 1e-8)\n    df['norm_sell_qty'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    return df\n\n# Anti-overfitting strategies\nclass AntiOverfitXGB:\n    \"\"\"\n    Implements multiple anti-overfitting strategies for XGBoost:\n    1. Adversarial training: Train models to overfit in opposite directions\n    2. Residual targeting: Train on residuals of overfit models\n    3. Feature/sample dropout: Random dropout during training\n    4. Prediction smoothing: Smooth predictions to reduce noise\n    \"\"\"\n    \n    def __init__(self, base_params=None):\n        self.base_params = base_params or {\n            'tree_method': 'hist',\n            'n_estimators': 500,\n            'learning_rate': 0.01,\n            'max_depth': 6,\n            'random_state': 42,\n            'n_jobs': -1,\n            'verbosity': 0\n        }\n        \n    def train_overfit_model(self, X, y, overfit_direction='high'):\n        \"\"\"Train a model designed to overfit in a specific direction\"\"\"\n        params = self.base_params.copy()\n        \n        if overfit_direction == 'high':\n            # Encourage overfitting with complex trees\n            params.update({\n                'max_depth': 12,\n                'min_child_weight': 1,\n                'subsample': 0.9,\n                'colsample_bytree': 0.9,\n                'reg_alpha': 0.001,\n                'reg_lambda': 0.001,\n                'learning_rate': 0.05\n            })\n        else:\n            # Encourage underfitting with simple trees\n            params.update({\n                'max_depth': 3,\n                'min_child_weight': 50,\n                'subsample': 0.5,\n                'colsample_bytree': 0.5,\n                'reg_alpha': 10,\n                'reg_lambda': 10,\n                'learning_rate': 0.001\n            })\n        \n        model = XGBRegressor(**params)\n        model.fit(X, y)\n        return model\n    \n    def identify_overfit_samples(self, X, y, n_folds=5):\n        \"\"\"Identify samples where model tends to overfit\"\"\"\n        kf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n        \n        errors = np.zeros(len(X))\n        predictions = np.zeros(len(X))\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n            X_fold_train = X.iloc[train_idx] if hasattr(X, 'iloc') else X[train_idx]\n            y_fold_train = y.iloc[train_idx] if hasattr(y, 'iloc') else y[train_idx]\n            X_fold_valid = X.iloc[valid_idx] if hasattr(X, 'iloc') else X[valid_idx]\n            y_fold_valid = y.iloc[valid_idx] if hasattr(y, 'iloc') else y[valid_idx]\n            \n            # Train overfit model\n            model = self.train_overfit_model(X_fold_train, y_fold_train, 'high')\n            \n            # Predict and calculate errors\n            pred = model.predict(X_fold_valid)\n            predictions[valid_idx] = pred\n            errors[valid_idx] = np.abs(pred - y_fold_valid)\n        \n        # Identify high-error samples (likely overfit)\n        error_threshold = np.percentile(errors, 75)\n        overfit_mask = errors > error_threshold\n        \n        return overfit_mask, predictions, errors\n    \n    def train_adversarial_ensemble(self, X, y, X_test, n_models=3):\n        \"\"\"Train ensemble with models that overfit in opposite directions\"\"\"\n        predictions_train = []\n        predictions_test = []\n        models = []\n        \n        # 1. Train high-overfitting model\n        print(\"Training high-overfitting model...\")\n        model_high = self.train_overfit_model(X, y, 'high')\n        pred_high_train = model_high.predict(X)\n        pred_high_test = model_high.predict(X_test)\n        predictions_train.append(pred_high_train)\n        predictions_test.append(pred_high_test)\n        models.append(('high', model_high))\n        \n        # 2. Train low-overfitting model\n        print(\"Training low-overfitting model...\")\n        model_low = self.train_overfit_model(X, y, 'low')\n        pred_low_train = model_low.predict(X)\n        pred_low_test = model_low.predict(X_test)\n        predictions_train.append(pred_low_train)\n        predictions_test.append(pred_low_test)\n        models.append(('low', model_low))\n        \n        # 3. Train on residuals\n        print(\"Training residual model...\")\n        residuals = y - (pred_high_train + pred_low_train) / 2\n        params = self.base_params.copy()\n        params.update({\n            'max_depth': 6,\n            'learning_rate': 0.02,\n            'subsample': 0.7,\n            'colsample_bytree': 0.7\n        })\n        model_residual = XGBRegressor(**params)\n        model_residual.fit(X, residuals)\n        pred_residual_train = model_residual.predict(X)\n        pred_residual_test = model_residual.predict(X_test)\n        \n        # Combine predictions\n        final_train = (pred_high_train + pred_low_train) / 2 + pred_residual_train\n        final_test = (pred_high_test + pred_low_test) / 2 + pred_residual_test\n        \n        return final_train, final_test, models\n    \n    def train_with_sample_weights(self, X, y, overfit_mask):\n        \"\"\"Train model with reduced weights on overfit-prone samples\"\"\"\n        # Create sample weights\n        sample_weights = np.ones(len(X))\n        sample_weights[overfit_mask] = 0.5  # Reduce weight on overfit samples\n        \n        params = self.base_params.copy()\n        params.update({\n            'max_depth': 8,\n            'learning_rate': 0.02,\n            'subsample': 0.8,\n            'colsample_bytree': 0.8,\n            'reg_alpha': 1,\n            'reg_lambda': 1\n        })\n        \n        model = XGBRegressor(**params)\n        model.fit(X, y, sample_weight=sample_weights)\n        return model\n    \n    def smooth_predictions(self, predictions, window=5):\n        \"\"\"Apply smoothing to reduce noise in predictions\"\"\"\n        from scipy.ndimage import gaussian_filter1d\n        return gaussian_filter1d(predictions, sigma=window/3)\n\n# Main pipeline\nprint(\"\\nLoading data...\")\ntrain = pd.read_parquet(CFG.train_path)\ntest = pd.read_parquet(CFG.test_path)\nsubmission = pd.read_csv(CFG.sample_sub_path)\n\nprint(f\"Train shape: {train.shape}\")\nprint(f\"Test shape: {test.shape}\")\n\n# Create 50 new proprietary X variables\ntrain = create_proprietary_x_variables(train)\ntest = create_proprietary_x_variables(test)\n\n# Feature engineering with extended features\ntrain = add_features(train)\ntest = add_features(test)\n\n# Memory optimization\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")\n\n# Select features - including more X features\nselected_x_features = [\n    \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n    \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n    \"X456\", \"X789\", \"X234\", \"X567\", \"X890\", \"X123\", \"X347\", \"X459\", \"X678\", \"X901\",\n    \"X112\", \"X223\", \"X334\", \"X445\", \"X556\", \"X667\", \"X778\", \"X889\", \"X990\", \"X101\"\n]\n\n# Add the 50 new proprietary X variables\nnew_x_features = [f\"X_prop_{i}\" for i in range(1, 51)]\nselected_x_features.extend(new_x_features)\n\n# Get ALL available X features (not just the selected ones)\nall_x_features = [col for col in train.columns if col.startswith('X') and col[1:].isdigit()]\nprint(f\"Total X features available: {len(all_x_features)}\")\n\n# Add more X features that aren't in our selected list\nadditional_x_features = [f for f in all_x_features if f not in selected_x_features]\n\n# Sort additional features numerically if possible\ndef extract_number(s):\n    try:\n        return int(s[1:])\n    except:\n        return float('inf')\n\nadditional_x_features.sort(key=extract_number)\n\n# Add up to CFG.max_x_features total X features\nremaining_slots = CFG.max_x_features - len(selected_x_features)\nif remaining_slots > 0:\n    selected_x_features.extend(additional_x_features[:remaining_slots])\n\navailable_x_features = [f for f in selected_x_features if f in train.columns]\nprint(f\"Using {len(available_x_features)} X features (including {len([f for f in available_x_features if 'prop' in f])} proprietary)\")\n\n# Market and engineered features\nmarket_features = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\nengineered_features = [col for col in train.columns if col in [\n    'bid_ask_spread', 'bid_ask_ratio', 'buy_sell_ratio', 'order_flow_imbalance',\n    'buying_pressure', 'selling_pressure', 'net_pressure', 'total_liquidity', 'liquidity_imbalance',\n    'liquidity_concentration', 'log_volume', 'sqrt_volume', 'volume_squared', 'inv_volume',\n    'kyle_lambda', 'vpin', 'roll_measure', 'effective_spread', 'realized_spread',\n    'price_impact', 'orderbook_slope', 'mid_quote_change', 'weighted_mid_price',\n    'trade_intensity', 'buy_intensity', 'sell_intensity', 'norm_bid_qty', 'norm_ask_qty',\n    'norm_buy_qty', 'norm_sell_qty'\n]]\n\n# Combine all base features\nbase_selected_features = market_features + available_x_features + engineered_features\nbase_selected_features = list(dict.fromkeys(base_selected_features))\nbase_selected_features = [f for f in base_selected_features if f in train.columns]\n\nprint(f\"\\nBase features before interactions: {len(base_selected_features)}\")\n\n# Create interaction features\ninteraction_df_train = create_feature_interactions(train, base_selected_features, CFG.n_interaction_features)\ninteraction_df_test = create_feature_interactions(test, base_selected_features, CFG.n_interaction_features)\n\n# Add interaction features to the dataframes\nfor col in interaction_df_train.columns:\n    train[col] = interaction_df_train[col]\n    test[col] = interaction_df_test[col]\n\n# Update selected features to include interactions\nselected_features = base_selected_features + list(interaction_df_train.columns)\nprint(f\"\\nTotal features including interactions: {len(selected_features)}\")\n\n# Prepare data\nX_train = train[selected_features]\ny_train = train['label']\nX_test = test[selected_features]\n\n# Scaling\nprint(\"\\nScaling features...\")\nscaler = StandardScaler()\nX_train_scaled = pd.DataFrame(\n    scaler.fit_transform(X_train),\n    columns=selected_features,\n    index=X_train.index\n)\nX_test_scaled = pd.DataFrame(\n    scaler.transform(X_test),\n    columns=selected_features,\n    index=X_test.index\n)\n\n# Initialize anti-overfitting trainer\nanti_overfit = AntiOverfitXGB()\n\n# Strategy 1: Identify overfit-prone samples\nprint(\"\\nIdentifying overfit-prone samples...\")\noverfit_mask, oof_predictions, errors = anti_overfit.identify_overfit_samples(\n    X_train_scaled, y_train, n_folds=5\n)\nprint(f\"Found {overfit_mask.sum()} overfit-prone samples ({100*overfit_mask.mean():.1f}%)\")\n\n# Strategy 2: Train adversarial ensemble\nprint(\"\\nTraining adversarial ensemble...\")\nadv_train, adv_test, adv_models = anti_overfit.train_adversarial_ensemble(\n    X_train_scaled, y_train, X_test_scaled\n)\n\n# Strategy 3: Train with sample weights\nprint(\"\\nTraining with adjusted sample weights...\")\nweighted_model = anti_overfit.train_with_sample_weights(\n    X_train_scaled, y_train, overfit_mask\n)\nweighted_train = weighted_model.predict(X_train_scaled)\nweighted_test = weighted_model.predict(X_test_scaled)\n\n# Strategy 4: Time-window ensemble\nprint(\"\\nTraining time-window models...\")\nwindows = [\n    {'name': 'very_recent', 'start': int(0.85 * len(train)), 'end': len(train)},\n    {'name': 'recent', 'start': int(0.7 * len(train)), 'end': int(0.85 * len(train))},\n    {'name': 'middle', 'start': int(0.5 * len(train)), 'end': int(0.7 * len(train))},\n    {'name': 'older', 'start': int(0.3 * len(train)), 'end': int(0.5 * len(train))},\n    {'name': 'old', 'start': int(0.1 * len(train)), 'end': int(0.3 * len(train))}\n]\n\nwindow_predictions_test = []\nfor window in windows:\n    print(f\"  Training on {window['name']} window...\")\n    X_window = X_train_scaled.iloc[window['start']:window['end']]\n    y_window = y_train.iloc[window['start']:window['end']]\n    \n    params = anti_overfit.base_params.copy()\n    params.update({\n        'max_depth': 7,\n        'learning_rate': 0.02,\n        'subsample': 0.75,\n        'colsample_bytree': 0.75\n    })\n    \n    model = XGBRegressor(**params)\n    model.fit(X_window, y_window)\n    window_predictions_test.append(model.predict(X_test_scaled))\n\n# Strategy 5: Additional ensemble with different parameters\nprint(\"\\nTraining additional ensemble models...\")\nadditional_models_test = []\n\n# Model with different tree structure\nparams1 = anti_overfit.base_params.copy()\nparams1.update({\n    'max_depth': 10,\n    'learning_rate': 0.015,\n    'subsample': 0.85,\n    'colsample_bytree': 0.85,\n    'min_child_weight': 3\n})\nmodel1 = XGBRegressor(**params1)\nmodel1.fit(X_train_scaled, y_train)\nadditional_models_test.append(model1.predict(X_test_scaled))\n\n# Model with heavy regularization\nparams2 = anti_overfit.base_params.copy()\nparams2.update({\n    'max_depth': 5,\n    'learning_rate': 0.01,\n    'subsample': 0.6,\n    'colsample_bytree': 0.6,\n    'reg_alpha': 5,\n    'reg_lambda': 5\n})\nmodel2 = XGBRegressor(**params2)\nmodel2.fit(X_train_scaled, y_train)\nadditional_models_test.append(model2.predict(X_test_scaled))\n\n# Combine all predictions\nprint(\"\\nCombining predictions...\")\n\n# Calculate validation scores\nadv_score = pearsonr(y_train, adv_train)[0]\nweighted_score = pearsonr(y_train, weighted_train)[0]\n\nprint(f\"\\nValidation scores:\")\nprint(f\"  Adversarial ensemble: {adv_score:.4f}\")\nprint(f\"  Weighted model: {weighted_score:.4f}\")\n\n# Final ensemble with weighted average\nfinal_test = (\n    0.35 * adv_test +                                    # Adversarial ensemble\n    0.25 * weighted_test +                               # Sample-weighted model\n    0.20 * np.mean(window_predictions_test, axis=0) +    # Time windows\n    0.20 * np.mean(additional_models_test, axis=0)      # Additional models\n)\n\n# Create submission\nsubmission['prediction'] = final_test\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"Submission saved to submission.csv\")\nprint(submission.head())\n\n# Save detailed predictions for analysis\npredictions_df = pd.DataFrame({\n    'adversarial': adv_test,\n    'weighted': weighted_test,\n    'window_avg': np.mean(window_predictions_test, axis=0),\n    'additional_avg': np.mean(additional_models_test, axis=0),\n    'final': final_test\n})\npredictions_df.to_csv('prediction_components.csv', index=False)\n\nprint(\"\\nPrediction statistics:\")\nprint(predictions_df.describe())\n\n# Print feature composition summary\nprint(\"\\nFeature composition summary:\")\nprint(f\"  Original X features: {len([f for f in selected_features if f.startswith('X') and 'prop' not in f and '_x_' not in f and '_div_' not in f and '_minus_' not in f and '_cubed' not in f and '_sqrt' not in f])}\")\nprint(f\"  Proprietary X features: {len([f for f in selected_features if 'X_prop_' in f])}\")\nprint(f\"  Market features: {len([f for f in selected_features if f in market_features])}\")\nprint(f\"  Engineered features: {len([f for f in selected_features if f in engineered_features])}\")\nprint(f\"  Interaction features: {len([f for f in selected_features if any(x in f for x in ['_x_', '_div_', '_minus_', '_cubed', '_sqrt_abs', '_x_log_'])])}\")\nprint(f\"  Total features: {len(selected_features)}\")\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"Extended pipeline completed successfully!\")\nprint(\"=\"*80)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-24T09:48:54.944732Z","iopub.execute_input":"2025-06-24T09:48:54.944991Z","iopub.status.idle":"2025-06-24T09:48:54.955514Z","shell.execute_reply.started":"2025-06-24T09:48:54.944969Z","shell.execute_reply":"2025-06-24T09:48:54.954331Z"}},"outputs":[],"execution_count":null}]}