{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":249869065,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.ensemble import ExtraTreesRegressor, RandomForestClassifier, RandomForestRegressor\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet\nfrom sklearn.model_selection import KFold, TimeSeriesSplit, cross_val_score\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.feature_selection import mutual_info_regression, f_regression, SelectKBest\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr, spearmanr, rankdata, ks_2samp\nimport gc\nimport os\nfrom datetime import datetime\nfrom typing import Dict, List, Tuple, Any, Optional\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport joblib\n\nprint(\"Starting Extra Trees Crypto Market Prediction...\")\nprint(\"=\" * 80)\n\n# Configuration with Extra Trees parameters\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    timestamp_recon_path = '/kaggle/input/the-order-of-the-test-rows-2/closest_rows.csv'\n    \n    n_folds = 3\n    random_state = 42\n    \n    # Feature engineering settings\n    n_proprietary_features = 25  # Slightly increased\n    n_interaction_features = 35  # Slightly increased\n    \n    # Time series settings\n    lag_periods = [1, 3, 5, 10, 20, 30, 60]\n    \n    # Important features from analysis\n    important_x_features = [\n        \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n        \"X344\", \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X137\", \"X174\", \"X178\",\n        \"X758\", \"X296\", \"X611\", \"X780\", \"X451\", \"X25\", \"X591\"\n    ]\n    \n    # Extra Trees parameters\n    n_estimators = 100  # Number of trees\n    max_depth = 15      # Max depth to prevent overfitting\n    min_samples_split = 20\n    min_samples_leaf = 10\n    max_features = 'sqrt'\n    \n    # Stability thresholds\n    stability_threshold = 0.3\n    importance_threshold = 0.001  # Lower for tree models\n    \n    # Training parameters\n    use_sample_weights = True\n    discriminator_weight = 0.1  # Reduced for trees\n    \n    # Feature selection parameters\n    min_features = 40\n    max_features = 120\n    \n    # Memory management\n    chunk_size = 50000\n    sample_size_for_selection = 100000\n\n# Memory optimization (same as before)\ndef reduce_mem_usage(df, name=\"\"):\n    \"\"\"Aggressive memory reduction\"\"\"\n    print(f\"Optimizing memory for {name}...\")\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            \n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n            else:\n                df[col] = df[col].astype(np.float32)\n    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage: {start_mem:.2f} MB -> {end_mem:.2f} MB ({100*(start_mem-end_mem)/start_mem:.1f}% reduction)')\n    \n    gc.collect()\n    return df\n\n# Enhanced market features\ndef add_market_features(df):\n    \"\"\"Create market microstructure features\"\"\"\n    print(\"Engineering market features...\")\n    \n    # Core features\n    df['bid_ask_spread'] = df['ask_qty'] - df['bid_qty']\n    df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-8)\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    # Liquidity features\n    df['total_liquidity'] = df['bid_qty'] + df['ask_qty']\n    df['liquidity_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_liquidity'] + 1e-8)\n    df['liquidity_ratio'] = df['total_liquidity'] / (df['volume'] + 1e-8)\n    \n    # Volume features\n    df['log_volume'] = np.log1p(df['volume'])\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    \n    # Microstructure\n    df['kyle_lambda'] = df['order_flow_imbalance'] / (df['sqrt_volume'] + 1e-8)\n    df['vpin'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    \n    # 5 key new features\n    df['amihud_illiquidity'] = np.abs(df['order_flow_imbalance']) / (df['volume'] * df['sqrt_volume'] + 1e-8)\n    df['roll_spread'] = 2 * np.sqrt(np.abs(df['order_flow_imbalance'] * df['kyle_lambda']))\n    df['info_share'] = (df['kyle_lambda'] ** 2) / (df['vpin'] + df['kyle_lambda'] ** 2 + 1e-8)\n    df['depth_imbalance'] = (df['bid_qty'] ** 2 - df['ask_qty'] ** 2) / ((df['bid_qty'] ** 2 + df['ask_qty'] ** 2) + 1e-8)\n    df['toxic_flow'] = df['vpin'] * df['kyle_lambda'] * np.abs(df['order_flow_imbalance'])\n    \n    # Additional useful features for trees\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['net_pressure'] = df['buying_pressure'] - df['selling_pressure']\n    df['price_impact'] = df['kyle_lambda'] * df['volume']\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    # Convert to float32\n    for col in df.columns:\n        if df[col].dtype == np.float64:\n            df[col] = df[col].astype(np.float32)\n    \n    return df\n\n# Enhanced proprietary features for trees\ndef create_proprietary_features(df, n_features=25):\n    \"\"\"Create proprietary features suitable for tree models\"\"\"\n    print(f\"Creating {n_features} proprietary features...\")\n    \n    x_features = [col for col in df.columns if col.startswith('X') and col[1:].isdigit()]\n    base_features = [f for f in CFG.important_x_features if f in df.columns][:15]\n    \n    if len(base_features) < 15:\n        x_variances = df[x_features].var()\n        high_var_features = x_variances.nlargest(15).index.tolist()\n        for feat in high_var_features:\n            if feat not in base_features:\n                base_features.append(feat)\n                if len(base_features) >= 15:\n                    break\n    \n    prop_idx = 1\n    \n    # Statistical features (good for trees)\n    if len(base_features) >= 10:\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].mean(axis=1).astype(np.float32)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].std(axis=1).astype(np.float32)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].max(axis=1).astype(np.float32)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].min(axis=1).astype(np.float32)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].median(axis=1).astype(np.float32)\n        prop_idx += 1\n        \n        # Quantiles\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].quantile(0.25, axis=1).astype(np.float32)\n        prop_idx += 1\n        df[f'X_prop_{prop_idx}'] = df[base_features[:10]].quantile(0.75, axis=1).astype(np.float32)\n        prop_idx += 1\n        \n        # Range\n        df[f'X_prop_{prop_idx}'] = (df[base_features[:10]].max(axis=1) - df[base_features[:10]].min(axis=1)).astype(np.float32)\n        prop_idx += 1\n    \n    # Non-linear transformations (trees can handle these well)\n    for i in range(min(10, n_features - prop_idx + 1)):\n        feat = base_features[i % len(base_features)]\n        if i % 4 == 0:\n            df[f'X_prop_{prop_idx}'] = np.sign(df[feat]) * np.sqrt(np.abs(df[feat])).astype(np.float32)\n        elif i % 4 == 1:\n            df[f'X_prop_{prop_idx}'] = np.log1p(np.abs(df[feat])).astype(np.float32)\n        elif i % 4 == 2:\n            df[f'X_prop_{prop_idx}'] = (df[feat] ** 2).astype(np.float32)\n        else:\n            df[f'X_prop_{prop_idx}'] = (rankdata(df[feat]) / len(df)).astype(np.float32)\n        prop_idx += 1\n    \n    # Market interactions\n    if 'volume' in df.columns and 'kyle_lambda' in df.columns:\n        for i in range(min(5, n_features - prop_idx + 1)):\n            feat = base_features[i % len(base_features)]\n            if i % 2 == 0:\n                df[f'X_prop_{prop_idx}'] = (df[feat] * np.log1p(df['volume'])).astype(np.float32)\n            else:\n                df[f'X_prop_{prop_idx}'] = (df[feat] * df['kyle_lambda']).astype(np.float32)\n            prop_idx += 1\n    \n    # Feature ratios\n    while prop_idx <= n_features:\n        i = prop_idx - 1\n        if i < len(base_features) - 1:\n            feat1 = base_features[i % len(base_features)]\n            feat2 = base_features[(i + 1) % len(base_features)]\n            df[f'X_prop_{prop_idx}'] = (df[feat1] / (np.abs(df[feat2]) + 1e-8)).astype(np.float32)\n        prop_idx += 1\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    \n    gc.collect()\n    return df\n\n# Enhanced interaction features for trees\ndef create_interaction_features(df, selected_features, n_interactions=35):\n    \"\"\"Create interaction features that work well with trees\"\"\"\n    print(f\"Creating {n_interactions} interaction features...\")\n    \n    x_features = [f for f in selected_features if f.startswith('X') and f in df.columns]\n    important_x = [f for f in CFG.important_x_features if f in x_features][:10]\n    \n    market_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'volume', \n                      'amihud_illiquidity', 'toxic_flow', 'liquidity_imbalance']\n    market_features = [f for f in market_features if f in df.columns][:7]\n    \n    interaction_dict = {}\n    \n    # X features with market microstructure\n    for i, x_feat in enumerate(important_x[:7]):\n        if len(interaction_dict) >= n_interactions // 2:\n            break\n        for j, market_feat in enumerate(market_features[:4]):\n            if len(interaction_dict) >= n_interactions // 2:\n                break\n            \n            feat_name = f'{x_feat}_x_{market_feat}'\n            interaction_dict[feat_name] = (df[x_feat] * df[market_feat]).astype(np.float32)\n    \n    # Polynomial features for important X\n    for i, feat in enumerate(important_x[:7]):\n        if len(interaction_dict) >= 3 * n_interactions // 4:\n            break\n        feat_name = f'{feat}_squared'\n        interaction_dict[feat_name] = (df[feat] ** 2).astype(np.float32)\n        \n        if i < 3:  # Cubic for top features\n            feat_name = f'{feat}_cubed'\n            interaction_dict[feat_name] = (df[feat] ** 3).astype(np.float32)\n    \n    # Market feature interactions\n    market_pairs = [\n        ('order_flow_imbalance', 'kyle_lambda'),\n        ('vpin', 'liquidity_imbalance'),\n        ('amihud_illiquidity', 'toxic_flow'),\n        ('volume', 'kyle_lambda')\n    ]\n    \n    for feat1, feat2 in market_pairs:\n        if len(interaction_dict) >= n_interactions:\n            break\n        if feat1 in df.columns and feat2 in df.columns:\n            feat_name = f'{feat1}_x_{feat2}'\n            interaction_dict[feat_name] = (df[feat1] * df[feat2]).astype(np.float32)\n    \n    # Create DataFrame\n    interaction_df = pd.DataFrame(interaction_dict, index=df.index)\n    \n    # Handle infinities and NaN\n    interaction_df = interaction_df.replace([np.inf, -np.inf], np.nan)\n    interaction_df = interaction_df.fillna(0)\n    \n    print(f\"Created {len(interaction_df.columns)} interaction features\")\n    gc.collect()\n    return interaction_df\n\n# Extra Trees Model with Discriminator\nclass ExtraTreesModelWithDiscriminator:\n    def __init__(self, n_estimators=100, max_depth=15, discriminator_weight=0.1):\n        self.n_estimators = n_estimators\n        self.max_depth = max_depth\n        self.discriminator_weight = discriminator_weight\n        self.model = None\n        self.discriminator = None\n        self.feature_names = None\n        \n    def train(self, X_train, y_train, X_test=None):\n        \"\"\"Train Extra Trees model with optional discriminator\"\"\"\n        print(f\"Training Extra Trees model (n_estimators={self.n_estimators}, max_depth={self.max_depth})...\")\n        \n        # Store feature names\n        self.feature_names = X_train.columns.tolist()\n        \n        # Create model\n        self.model = ExtraTreesRegressor(\n            n_estimators=self.n_estimators,\n            max_depth=self.max_depth,\n            min_samples_split=CFG.min_samples_split,\n            min_samples_leaf=CFG.min_samples_leaf,\n            max_features=CFG.max_features,\n            random_state=CFG.random_state,\n            n_jobs=-1\n        )\n        \n        # Train with or without discriminator\n        if X_test is None or self.discriminator_weight == 0:\n            self.model.fit(X_train, y_train)\n        else:\n            # Use discriminator for sample weighting\n            print(\"  Training discriminator...\")\n            \n            # Sample for discriminator\n            n_sample = min(30000, len(X_train))\n            idx_train = np.random.choice(len(X_train), n_sample, replace=False)\n            idx_test = np.random.choice(len(X_test), n_sample, replace=False)\n            \n            X_disc = np.vstack([X_train.iloc[idx_train], X_test.iloc[idx_test]])\n            y_disc = np.hstack([np.ones(n_sample), np.zeros(n_sample)])\n            \n            # Train discriminator\n            self.discriminator = RandomForestClassifier(\n                n_estimators=50,\n                max_depth=5,\n                random_state=42,\n                n_jobs=-1\n            )\n            self.discriminator.fit(X_disc, y_disc)\n            \n            # Get sample weights\n            train_disc_scores = self.discriminator.predict_proba(X_train)[:, 1]\n            sample_weights = 1 - self.discriminator_weight * train_disc_scores\n            sample_weights = np.clip(sample_weights, 0.1, 1.0)\n            \n            # Train with weights\n            print(\"  Training with sample weights...\")\n            self.model.fit(X_train, y_train, sample_weight=sample_weights)\n            \n            del self.discriminator, X_disc, y_disc\n            gc.collect()\n        \n        return self.model.predict(X_train)\n    \n    def predict(self, X):\n        \"\"\"Make predictions\"\"\"\n        return self.model.predict(X)\n    \n    def get_feature_importance(self):\n        \"\"\"Get feature importance from Extra Trees\"\"\"\n        if hasattr(self.model, 'feature_importances_'):\n            return pd.Series(\n                self.model.feature_importances_,\n                index=self.feature_names\n            ).sort_values(ascending=False)\n        return None\n\n# Feature Analyzer for Tree Models\nclass TreeFeatureAnalyzer:\n    def __init__(self):\n        self.train_importance = None\n        self.test_importance = None\n        self.stability_metrics = None\n        \n    def analyze_importance(self, X_train, y_train, X_test, y_test_pseudo):\n        \"\"\"Analyze feature importance for tree models\"\"\"\n        print(\"\\nAnalyzing feature importance with Extra Trees...\")\n        \n        # Train model\n        model_train = ExtraTreesModelWithDiscriminator(\n            n_estimators=50,  # Fewer trees for speed\n            max_depth=10\n        )\n        model_train.train(X_train, y_train)\n        self.train_importance = model_train.get_feature_importance()\n        \n        # Test model (on subset)\n        n_test = min(50000, len(X_test))\n        idx = np.random.choice(len(X_test), n_test, replace=False)\n        \n        model_test = ExtraTreesModelWithDiscriminator(\n            n_estimators=50,\n            max_depth=10\n        )\n        model_test.train(X_test.iloc[idx], y_test_pseudo.iloc[idx])\n        self.test_importance = model_test.get_feature_importance()\n        \n        return self\n    \n    def calculate_stability(self):\n        \"\"\"Calculate stability metrics for tree-based importance\"\"\"\n        if self.train_importance is None or self.test_importance is None:\n            return self\n        \n        print(\"\\nCalculating feature stability...\")\n        \n        all_features = list(set(self.train_importance.index) | set(self.test_importance.index))\n        \n        stability_data = []\n        for feature in all_features:\n            train_imp = self.train_importance.get(feature, 0)\n            test_imp = self.test_importance.get(feature, 0)\n            \n            # Stability score for tree importance\n            if train_imp > 0 and test_imp > 0:\n                # Use harmonic mean for stability\n                stability_score = 2 * train_imp * test_imp / (train_imp + test_imp)\n            else:\n                stability_score = 0\n            \n            stability_data.append({\n                'feature': feature,\n                'train_importance': train_imp,\n                'test_importance': test_imp,\n                'stability_score': stability_score\n            })\n        \n        self.stability_metrics = pd.DataFrame(stability_data)\n        self.stability_metrics = self.stability_metrics.sort_values('stability_score', ascending=False)\n        \n        return self\n    \n    def get_features_to_remove(self, n_remove=10):\n        \"\"\"Get features to remove based on tree importance\"\"\"\n        if self.stability_metrics is None:\n            return []\n        \n        # Remove features with low stability or very low importance\n        candidates = self.stability_metrics[\n            (self.stability_metrics['stability_score'] < CFG.stability_threshold * 0.001) |  # Adjusted for tree importance scale\n            (self.stability_metrics['train_importance'] < CFG.importance_threshold)\n        ]\n        \n        # Also consider features with extreme importance differences\n        candidates2 = self.stability_metrics[\n            (self.stability_metrics['train_importance'] > 5 * self.stability_metrics['test_importance']) |\n            (self.stability_metrics['test_importance'] > 5 * self.stability_metrics['train_importance'])\n        ]\n        \n        all_candidates = pd.concat([candidates, candidates2]).drop_duplicates()\n        \n        return all_candidates.head(n_remove)['feature'].tolist()\n\n# Enhanced Feature Selector for Trees\nclass TreeFeatureSelector:\n    def __init__(self):\n        self.selected_features = None\n        self.feature_scores = None\n        \n    def select_features(self, X, y, n_features=None):\n        \"\"\"Feature selection optimized for tree models\"\"\"\n        print(\"\\nFeature selection for Extra Trees...\")\n        \n        # Use subset for faster selection\n        n_sample = min(CFG.sample_size_for_selection, len(X))\n        idx = np.random.choice(len(X), n_sample, replace=False)\n        X_sample = X.iloc[idx]\n        y_sample = y.iloc[idx]\n        \n        print(\"  Calculating feature importance...\")\n        \n        # 1. Extra Trees importance\n        et_model = ExtraTreesRegressor(\n            n_estimators=50,\n            max_depth=10,\n            random_state=42,\n            n_jobs=-1\n        )\n        et_model.fit(X_sample, y_sample)\n        et_scores = pd.Series(et_model.feature_importances_, index=X.columns)\n        \n        # 2. Random Forest importance (for comparison)\n        rf_model = RandomForestRegressor(\n            n_estimators=50,\n            max_depth=10,\n            random_state=42,\n            n_jobs=-1\n        )\n        rf_model.fit(X_sample, y_sample)\n        rf_scores = pd.Series(rf_model.feature_importances_, index=X.columns)\n        \n        # 3. Mutual information\n        mi_sample = min(20000, n_sample)\n        mi_idx = np.random.choice(n_sample, mi_sample, replace=False)\n        mi_scores = mutual_info_regression(\n            X_sample.iloc[mi_idx], \n            y_sample.iloc[mi_idx], \n            random_state=42\n        )\n        mi_scores = pd.Series(mi_scores, index=X.columns)\n        \n        # 4. Simple correlation (still useful)\n        corr_scores = pd.Series({\n            col: abs(pearsonr(X_sample[col], y_sample)[0]) \n            for col in X_sample.columns\n        })\n        \n        # Normalize scores\n        et_scores = (et_scores - et_scores.min()) / (et_scores.max() - et_scores.min() + 1e-8)\n        rf_scores = (rf_scores - rf_scores.min()) / (rf_scores.max() - rf_scores.min() + 1e-8)\n        mi_scores = (mi_scores - mi_scores.min()) / (mi_scores.max() - mi_scores.min() + 1e-8)\n        corr_scores = (corr_scores - corr_scores.min()) / (corr_scores.max() - corr_scores.min() + 1e-8)\n        \n        # Combined score (weighted towards tree-based methods)\n        combined_scores = 0.4 * et_scores + 0.3 * rf_scores + 0.2 * mi_scores + 0.1 * corr_scores\n        \n        self.feature_scores = pd.DataFrame({\n            'extra_trees': et_scores,\n            'random_forest': rf_scores,\n            'mutual_info': mi_scores,\n            'correlation': corr_scores,\n            'combined': combined_scores\n        }).sort_values('combined', ascending=False)\n        \n        # Select features\n        if n_features is None:\n            n_features = min(CFG.max_features, max(CFG.min_features, int(len(X.columns) * 0.6)))\n        \n        self.selected_features = self.feature_scores.head(n_features).index.tolist()\n        \n        # Ensure critical features\n        critical_features = ['order_flow_imbalance', 'kyle_lambda', 'vpin', 'volume', \n                           'amihud_illiquidity', 'toxic_flow', 'liquidity_imbalance']\n        for feat in critical_features:\n            if feat in X.columns and feat not in self.selected_features:\n                self.selected_features.append(feat)\n        \n        print(f\"Selected {len(self.selected_features)} features\")\n        \n        # Clean up\n        del X_sample, y_sample, et_model, rf_model\n        gc.collect()\n        \n        return self.selected_features\n\n# Main pipeline with Extra Trees\ndef main():\n    print(\"\\nLoading data...\")\n    train_df = pd.read_parquet(CFG.train_path)\n    test_df = pd.read_parquet(CFG.test_path)\n    submission = pd.read_csv(CFG.sample_sub_path)\n    \n    print(f\"Train shape: {train_df.shape}\")\n    print(f\"Test shape: {test_df.shape}\")\n    \n    # Memory optimization\n    train_df = reduce_mem_usage(train_df, \"train\")\n    test_df = reduce_mem_usage(test_df, \"test\")\n    \n    # Extract labels\n    y_train = train_df['label'].astype(np.float32)\n    \n    # Drop less important X features\n    x_cols = [col for col in train_df.columns if col.startswith('X') and col[1:].isdigit()]\n    important_x = CFG.important_x_features + [f'X{i}' for i in range(1, 31)]  # Keep top 30 more\n    drop_x = [col for col in x_cols if col not in important_x]\n    \n    print(f\"Dropping {len(drop_x)} less important X features...\")\n    train_df = train_df.drop(columns=drop_x)\n    test_df = test_df.drop(columns=drop_x)\n    gc.collect()\n    \n    # Feature engineering\n    train_df = add_market_features(train_df)\n    test_df = add_market_features(test_df)\n    gc.collect()\n    \n    train_df = create_proprietary_features(train_df, CFG.n_proprietary_features)\n    test_df = create_proprietary_features(test_df, CFG.n_proprietary_features)\n    gc.collect()\n    \n    # Get feature columns\n    feature_cols = [col for col in train_df.columns if col != 'label' and col != 'timestamp']\n    \n    # Create interaction features\n    interaction_train = create_interaction_features(train_df, feature_cols, CFG.n_interaction_features)\n    interaction_test = create_interaction_features(test_df, feature_cols, CFG.n_interaction_features)\n    \n    # Add interaction features\n    train_df = pd.concat([train_df, interaction_train], axis=1)\n    test_df = pd.concat([test_df, interaction_test], axis=1)\n    \n    del interaction_train, interaction_test\n    gc.collect()\n    \n    # Update feature columns\n    feature_cols = [col for col in train_df.columns if col != 'label' and col != 'timestamp']\n    \n    print(f\"\\nTotal features created: {len(feature_cols)}\")\n    \n    # Step 1: Feature selection\n    print(\"\\n\" + \"=\"*60)\n    print(\"STEP 1: FEATURE SELECTION FOR EXTRA TREES\")\n    print(\"=\"*60)\n    \n    X_train = train_df[feature_cols]\n    X_test = test_df[feature_cols]\n    \n    selector = TreeFeatureSelector()\n    selected_features = selector.select_features(X_train, y_train)\n    \n    X_train_selected = X_train[selected_features]\n    X_test_selected = X_test[selected_features]\n    \n    # Clean up\n    del X_train, X_test\n    gc.collect()\n    \n    # Step 2: Train baseline Extra Trees model\n    print(\"\\n\" + \"=\"*60)\n    print(\"STEP 2: BASELINE EXTRA TREES MODEL\")\n    print(\"=\"*60)\n    \n    baseline_model = ExtraTreesModelWithDiscriminator(\n        n_estimators=CFG.n_estimators,\n        max_depth=CFG.max_depth,\n        discriminator_weight=CFG.discriminator_weight\n    )\n    baseline_pred_train = baseline_model.train(X_train_selected, y_train, X_test_selected)\n    baseline_pred_test = baseline_model.predict(X_test_selected)\n    \n    baseline_score = pearsonr(y_train, baseline_pred_train)[0]\n    print(f\"\\nBaseline Extra Trees Performance: {baseline_score:.4f}\")\n    \n    # Step 3: Generate pseudo labels with ensemble\n    print(\"\\n\" + \"=\"*60)\n    print(\"STEP 3: PSEUDO LABELING WITH ENSEMBLE\")\n    print(\"=\"*60)\n    \n    # Use multiple configurations for pseudo labeling\n    pseudo_preds = []\n    configs = [\n        (100, 15),  # (n_estimators, max_depth)\n        (80, 12),\n        (60, 18)\n    ]\n    \n    for n_est, max_d in configs:\n        print(f\"  Training Extra Trees ({n_est} trees, depth {max_d})...\")\n        model = ExtraTreesModelWithDiscriminator(\n            n_estimators=n_est,\n            max_depth=max_d,\n            discriminator_weight=0.05\n        )\n        model.train(X_train_selected, y_train)\n        pred = model.predict(X_test_selected)\n        pseudo_preds.append(pred)\n    \n    y_test_pseudo = pd.Series(np.mean(pseudo_preds, axis=0), index=test_df.index)\n    \n    print(f\"\\nPseudo label statistics:\")\n    print(f\"  Mean: {y_test_pseudo.mean():.4f}\")\n    print(f\"  Std: {y_test_pseudo.std():.4f}\")\n    \n    # Step 4: Feature stability analysis\n    print(\"\\n\" + \"=\"*60)\n    print(\"STEP 4: TREE-BASED STABILITY ANALYSIS\")\n    print(\"=\"*60)\n    \n    analyzer = TreeFeatureAnalyzer()\n    analyzer.analyze_importance(X_train_selected, y_train, X_test_selected, y_test_pseudo)\n    analyzer.calculate_stability()\n    \n    features_to_remove = analyzer.get_features_to_remove(n_remove=min(15, len(selected_features) // 4))\n    \n    # Step 5: Retrain with refined features\n    if features_to_remove:\n        print(\"\\n\" + \"=\"*60)\n        print(\"STEP 5: REFINED EXTRA TREES MODEL\")\n        print(\"=\"*60)\n        \n        refined_features = [f for f in selected_features if f not in features_to_remove]\n        print(f\"Refined features: {len(selected_features)} -> {len(refined_features)}\")\n        \n        X_train_refined = X_train_selected[refined_features]\n        X_test_refined = X_test_selected[refined_features]\n        \n        # Train refined model with more trees\n        refined_model = ExtraTreesModelWithDiscriminator(\n            n_estimators=CFG.n_estimators + 50,  # More trees for refined model\n            max_depth=CFG.max_depth,\n            discriminator_weight=CFG.discriminator_weight\n        )\n        refined_pred_train = refined_model.train(X_train_refined, y_train, X_test_refined)\n        refined_pred_test = refined_model.predict(X_test_refined)\n        \n        refined_score = pearsonr(y_train, refined_pred_train)[0]\n        print(f\"Refined Extra Trees Performance: {refined_score:.4f} (Δ: {refined_score - baseline_score:+.4f})\")\n        \n        # Final ensemble\n        print(\"\\n\" + \"=\"*60)\n        print(\"STEP 6: FINAL ENSEMBLE\")\n        print(\"=\"*60)\n        \n        ensemble_preds = []\n        ensemble_weights = []\n        \n        # Different tree configurations\n        ensemble_configs = [\n            (150, 15, 'sqrt'),   # (n_estimators, max_depth, max_features)\n            (100, 20, 'sqrt'),\n            (120, 12, 0.8)\n        ]\n        \n        for n_est, max_d, max_feat in ensemble_configs:\n            print(f\"  Training ensemble model ({n_est} trees, depth {max_d})...\")\n            \n            model = ExtraTreesRegressor(\n                n_estimators=n_est,\n                max_depth=max_d,\n                max_features=max_feat,\n                min_samples_split=CFG.min_samples_split,\n                min_samples_leaf=CFG.min_samples_leaf,\n                random_state=CFG.random_state,\n                n_jobs=-1\n            )\n            model.fit(X_train_refined, y_train)\n            \n            train_pred = model.predict(X_train_refined)\n            test_pred = model.predict(X_test_refined)\n            \n            score = pearsonr(y_train, train_pred)[0]\n            ensemble_preds.append(test_pred)\n            ensemble_weights.append(score)\n        \n        # Normalize weights\n        ensemble_weights = np.array(ensemble_weights)\n        ensemble_weights = ensemble_weights / ensemble_weights.sum()\n        \n        print(f\"\\nEnsemble weights: {ensemble_weights}\")\n        \n        final_predictions = np.average(ensemble_preds, axis=0, weights=ensemble_weights)\n    else:\n        print(\"\\nNo features removed - using baseline\")\n        refined_pred_test = baseline_pred_test\n        final_predictions = baseline_pred_test\n    \n    # Create submissions\n    print(\"\\n\" + \"=\"*60)\n    print(\"CREATING SUBMISSIONS\")\n    print(\"=\"*60)\n    \n    submission['prediction'] = baseline_pred_test\n    submission.to_csv('submission_baseline_extratrees.csv', index=False)\n    print(\"Created: submission_baseline_extratrees.csv\")\n    \n    if 'refined_pred_test' in locals():\n        submission['prediction'] = refined_pred_test\n        submission.to_csv('submission_refined_extratrees.csv', index=False)\n        print(\"Created: submission_refined_extratrees.csv\")\n        \n        submission['prediction'] = final_predictions\n        submission.to_csv('submission_ensemble_extratrees.csv', index=False)\n        print(\"Created: submission_ensemble_extratrees.csv\")\n    \n    # Save feature importance\n    print(\"\\nSaving feature importance analysis...\")\n    feature_importance_df = baseline_model.get_feature_importance().head(50)\n    feature_importance_df.to_csv('feature_importance_extratrees.csv')\n    print(\"Created: feature_importance_extratrees.csv\")\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"EXTRA TREES PIPELINE COMPLETED!\")\n    print(\"=\"*80)\n    print(\"\\nExpected improvements over linear models:\")\n    print(\"- Better handling of non-linear relationships\")\n    print(\"- More robust to outliers\")\n    print(\"- Can capture complex feature interactions\")\n    print(\"- Generally higher correlation scores on complex data\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}