{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.stats import pearsonr\nimport warnings\nwarnings.filterwarnings('ignore')\nimport gc\nfrom typing import Dict, List, Tuple, Optional\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport time\n\n# ===== Feature Engineering =====\ndef feature_engineering(df):\n    \"\"\"Original features plus new robust features\"\"\"\n    # Convert to float32 for memory efficiency\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n    for col in numeric_cols:\n        df[col] = df[col].astype(np.float32)\n    \n    # Original features\n    df['volume_weighted_sell'] = (df['sell_qty'] * df['volume']).astype(np.float32)\n    df['buy_sell_ratio'] = (df['buy_qty'] / (df['sell_qty'] + 1e-8)).astype(np.float32)\n    df['selling_pressure'] = (df['sell_qty'] / (df['volume'] + 1e-8)).astype(np.float32)\n    df['effective_spread_proxy'] = (np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)).astype(np.float32)\n    \n    # New robust features\n    df['log_volume'] = np.log1p(df['volume']).astype(np.float32)\n    df['bid_ask_imbalance'] = ((df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-8)).astype(np.float32)\n    df['order_flow_imbalance'] = ((df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)).astype(np.float32)\n    df['liquidity_ratio'] = ((df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-8)).astype(np.float32)\n    \n    return df\n\n# ===== Configuration =====\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    # Core features\n    CORE_FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\n        \"bid_qty\", \"ask_qty\"\n    ]\n    \n    # Engineered features\n    ENGINEERED_FEATURES = [\n        'volume_weighted_sell', 'buy_sell_ratio', 'selling_pressure', 'effective_spread_proxy',\n        'log_volume', 'bid_ask_imbalance', 'order_flow_imbalance', 'liquidity_ratio'\n    ]\n    \n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n    \n    # GPU availability check\n    USE_GPU = False  # Set to False for CPU-only execution\n\n# ===== Model Parameters =====\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"cpu\",  # Changed to CPU for compatibility\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\nLGBM_PARAMS = {\n    \"n_estimators\": 1000,\n    \"learning_rate\": 0.025,\n    \"num_leaves\": 15,\n    \"min_child_samples\": 30,\n    \"subsample\": 0.1,\n    \"colsample_bytree\": 0.7,\n    \"reg_alpha\": 20,\n    \"reg_lambda\": 50,\n    \"random_state\": Config.RANDOM_STATE,\n    \"device\": \"cpu\",\n    \"verbosity\": -1,\n    \"n_jobs\": 4\n}\n\n# ===== Performance Tracker =====\nclass PerformanceTracker:\n    \"\"\"Track model performance and generate insights\"\"\"\n    \n    def __init__(self):\n        self.baseline_score = None\n        self.results = []\n        self.feature_importance = {}\n        self.insights = []\n        \n    def record_baseline(self, score: float, model_type: str):\n        \"\"\"Record baseline performance\"\"\"\n        self.baseline_score = score\n        print(f\"\\n📊 BASELINE ESTABLISHED\")\n        print(f\"   Model: {model_type}\")\n        print(f\"   Score: {score:.6f}\")\n        print(\"=\"*60)\n        \n    def record_feature_test(self, feature: str, score: float, model_type: str, improvement: float):\n        \"\"\"Record single feature addition test\"\"\"\n        self.results.append({\n            'feature': feature,\n            'score': score,\n            'model_type': model_type,\n            'improvement': improvement\n        })\n        \n        # Update feature importance\n        if feature not in self.feature_importance:\n            self.feature_importance[feature] = []\n        self.feature_importance[feature].append(improvement)\n        \n        # Generate insight\n        if improvement > 0.001:\n            self.insights.append(f\"✅ {feature} improves {model_type} by {improvement:.4f}\")\n        elif improvement < -0.001:\n            self.insights.append(f\"❌ {feature} degrades {model_type} by {improvement:.4f}\")\n            \n    def get_top_features(self, n: int = 5) -> List[str]:\n        \"\"\"Get top performing features based on average improvement\"\"\"\n        avg_improvements = {}\n        for feature, improvements in self.feature_importance.items():\n            avg_improvements[feature] = np.mean(improvements)\n            \n        sorted_features = sorted(avg_improvements.items(), key=lambda x: x[1], reverse=True)\n        return [feat for feat, _ in sorted_features[:n]]\n    \n    def print_summary(self):\n        \"\"\"Print comprehensive performance summary\"\"\"\n        print(\"\\n\" + \"=\"*80)\n        print(\"🔬 PERFORMANCE ANALYSIS SUMMARY\")\n        print(\"=\"*80)\n        \n        if not self.results:\n            print(\"No results recorded yet.\")\n            return\n            \n        # Best overall result\n        best_result = max(self.results, key=lambda x: x['score'])\n        print(f\"\\n🏆 BEST RESULT\")\n        print(f\"   Feature: {best_result['feature']}\")\n        print(f\"   Model: {best_result['model_type']}\")\n        print(f\"   Score: {best_result['score']:.6f}\")\n        print(f\"   Improvement: {best_result['improvement']:.6f} ({best_result['improvement']/self.baseline_score*100:.2f}%)\")\n        \n        # Top features by average improvement\n        print(f\"\\n📈 TOP FEATURES BY AVERAGE IMPROVEMENT\")\n        top_features = self.get_top_features(10)\n        for i, feature in enumerate(top_features[:5], 1):\n            avg_imp = np.mean(self.feature_importance[feature])\n            print(f\"   {i}. {feature}: {avg_imp:.6f} avg improvement\")\n            \n        # Model-specific insights\n        print(f\"\\n🎯 MODEL-SPECIFIC INSIGHTS\")\n        for model_type in ['XGBoost', 'LightGBM', 'Neural Network']:\n            model_results = [r for r in self.results if r['model_type'] == model_type]\n            if model_results:\n                best_for_model = max(model_results, key=lambda x: x['improvement'])\n                print(f\"   {model_type}: Best with {best_for_model['feature']} (+{best_for_model['improvement']:.6f})\")\n                \n        # Key insights\n        if self.insights:\n            print(f\"\\n💡 KEY INSIGHTS\")\n            for insight in self.insights[-5:]:  # Show last 5 insights\n                print(f\"   {insight}\")\n\n# ===== Bucket Transformer =====\nclass BucketTransformer:\n    \"\"\"Transform features into 3 buckets based on quantiles\"\"\"\n    \n    def __init__(self):\n        self.thresholds = {}\n        \n    def fit(self, X: pd.DataFrame, features: List[str]):\n        \"\"\"Fit bucket thresholds based on data distribution\"\"\"\n        for feature in features:\n            if feature in X.columns:\n                # Calculate tertile thresholds\n                q33 = X[feature].quantile(0.33)\n                q67 = X[feature].quantile(0.67)\n                self.thresholds[feature] = (q33, q67)\n                \n    def transform(self, X: pd.DataFrame, feature: str) -> pd.Series:\n        \"\"\"Transform a single feature into 3 buckets\"\"\"\n        if feature not in self.thresholds:\n            raise ValueError(f\"Feature {feature} not fitted\")\n            \n        q33, q67 = self.thresholds[feature]\n        \n        # Create buckets: 1, 2, 3\n        buckets = pd.Series(2, index=X.index, dtype=np.int8)  # Default to middle bucket\n        buckets[X[feature] <= q33] = 1\n        buckets[X[feature] > q67] = 3\n        \n        return buckets\n\n# ===== Neural Network Models =====\ndef create_balanced_nn(input_dim: int) -> keras.Model:\n    \"\"\"Create a balanced neural network\"\"\"\n    model = keras.Sequential([\n        layers.Input(shape=(input_dim,)),\n        layers.Dense(128, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.3),\n        layers.Dense(64, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.2),\n        layers.Dense(32, activation='relu'),\n        layers.Dense(1)\n    ])\n    \n    model.compile(\n        optimizer=keras.optimizers.Adam(learning_rate=0.001),\n        loss='mse'\n    )\n    \n    return model\n\ndef create_wide_nn(input_dim: int) -> keras.Model:\n    \"\"\"Create a wide neural network\"\"\"\n    model = keras.Sequential([\n        layers.Input(shape=(input_dim,)),\n        layers.Dense(256, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.3),\n        layers.Dense(128, activation='relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.2),\n        layers.Dense(1)\n    ])\n    \n    model.compile(\n        optimizer=keras.optimizers.Adam(learning_rate=0.001),\n        loss='mse'\n    )\n    \n    return model\n\n# ===== VAE Implementation =====\nclass VAE(keras.Model):\n    \"\"\"Variational Autoencoder for feature reduction\"\"\"\n    \n    def __init__(self, input_dim: int, latent_dim: int = 10):\n        super(VAE, self).__init__()\n        self.latent_dim = latent_dim\n        \n        # Encoder\n        self.encoder = keras.Sequential([\n            layers.Input(shape=(input_dim,)),\n            layers.Dense(64, activation='relu'),\n            layers.Dense(32, activation='relu')\n        ])\n        \n        self.z_mean = layers.Dense(latent_dim)\n        self.z_log_var = layers.Dense(latent_dim)\n        \n        # Decoder\n        self.decoder = keras.Sequential([\n            layers.Input(shape=(latent_dim,)),\n            layers.Dense(32, activation='relu'),\n            layers.Dense(64, activation='relu'),\n            layers.Dense(input_dim, activation='sigmoid')\n        ])\n        \n    def encode(self, x):\n        h = self.encoder(x)\n        return self.z_mean(h), self.z_log_var(h)\n    \n    def reparameterize(self, z_mean, z_log_var):\n        eps = tf.random.normal(shape=tf.shape(z_mean))\n        return z_mean + tf.exp(0.5 * z_log_var) * eps\n    \n    def decode(self, z):\n        return self.decoder(z)\n    \n    def call(self, inputs):\n        z_mean, z_log_var = self.encode(inputs)\n        z = self.reparameterize(z_mean, z_log_var)\n        reconstructed = self.decode(z)\n        \n        # Add KL divergence loss\n        kl_loss = -0.5 * tf.reduce_mean(\n            1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var)\n        )\n        self.add_loss(kl_loss)\n        \n        return reconstructed\n    \n    def get_latent_features(self, x):\n        \"\"\"Get latent space representation\"\"\"\n        z_mean, _ = self.encode(x)\n        return z_mean\n\n# ===== Model Training Functions =====\ndef train_model_cv(X: pd.DataFrame, y: pd.Series, model_type: str, features: List[str]) -> Tuple[float, np.ndarray, np.ndarray]:\n    \"\"\"Train model with cross-validation\"\"\"\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    oof_preds = np.zeros(len(X))\n    cv_scores = []\n    \n    X_features = X[features].values\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n        X_train, X_valid = X_features[train_idx], X_features[valid_idx]\n        y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n        \n        if model_type == \"XGBoost\":\n            model = XGBRegressor(**XGB_PARAMS)\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False)\n            valid_preds = model.predict(X_valid)\n            \n        elif model_type == \"LightGBM\":\n            model = LGBMRegressor(**LGBM_PARAMS)\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], callbacks=[])\n            valid_preds = model.predict(X_valid)\n            \n        elif model_type in [\"Balanced NN\", \"Wide NN\"]:\n            # Scale features for neural networks\n            scaler = StandardScaler()\n            X_train_scaled = scaler.fit_transform(X_train)\n            X_valid_scaled = scaler.transform(X_valid)\n            \n            if model_type == \"Balanced NN\":\n                model = create_balanced_nn(X_train.shape[1])\n            else:\n                model = create_wide_nn(X_train.shape[1])\n                \n            # Train with early stopping\n            model.fit(\n                X_train_scaled, y_train,\n                validation_data=(X_valid_scaled, y_valid),\n                epochs=50,\n                batch_size=256,\n                callbacks=[\n                    keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)\n                ],\n                verbose=0\n            )\n            \n            valid_preds = model.predict(X_valid_scaled).flatten()\n            \n        oof_preds[valid_idx] = valid_preds\n        fold_score = pearsonr(y_valid, valid_preds)[0]\n        cv_scores.append(fold_score)\n    \n    mean_score = np.mean(cv_scores)\n    return mean_score, oof_preds, np.array(cv_scores)\n\n# ===== Feature Exploration System =====\nclass FeatureExplorer:\n    \"\"\"Systematically explore feature additions\"\"\"\n    \n    def __init__(self, train_df: pd.DataFrame, test_df: pd.DataFrame):\n        self.train_df = train_df\n        self.test_df = test_df\n        self.tracker = PerformanceTracker()\n        self.bucket_transformer = BucketTransformer()\n        self.baseline_features = Config.CORE_FEATURES + Config.ENGINEERED_FEATURES\n        \n        # Get all X features not in core features\n        all_x_features = [col for col in train_df.columns if col.startswith('X') and col not in Config.CORE_FEATURES]\n        self.candidate_features = sorted(all_x_features)\n        \n        print(f\"🔍 FEATURE EXPLORATION SETUP\")\n        print(f\"   Baseline features: {len(self.baseline_features)}\")\n        print(f\"   Candidate features for bucketing: {len(self.candidate_features)}\")\n        print(\"=\"*60)\n        \n    def establish_baseline(self):\n        \"\"\"Establish baseline performance with core + engineered features\"\"\"\n        print(\"\\n📊 ESTABLISHING BASELINE PERFORMANCE\")\n        print(\"=\"*60)\n        \n        y = self.train_df[Config.LABEL_COLUMN]\n        \n        for model_type in [\"XGBoost\", \"LightGBM\", \"Balanced NN\", \"Wide NN\"]:\n            print(f\"\\nTraining {model_type} baseline...\")\n            start_time = time.time()\n            \n            score, _, _ = train_model_cv(self.train_df, y, model_type, self.baseline_features)\n            \n            elapsed = time.time() - start_time\n            print(f\"   Score: {score:.6f} (took {elapsed:.1f}s)\")\n            \n            if model_type == \"XGBoost\":  # Use XGBoost as primary baseline\n                self.tracker.record_baseline(score, model_type)\n                self.baseline_score = score\n                \n    def explore_single_features(self, n_features: int = 5):\n        \"\"\"Test adding single bucketed features\"\"\"\n        print(f\"\\n🔬 EXPLORING TOP {n_features} BUCKETED FEATURES\")\n        print(\"=\"*60)\n        \n        # Fit bucket transformer on all candidate features\n        print(\"Fitting bucket transformer...\")\n        self.bucket_transformer.fit(self.train_df, self.candidate_features)\n        \n        y = self.train_df[Config.LABEL_COLUMN]\n        \n        # Test each feature\n        feature_scores = []\n        \n        for i, feature in enumerate(self.candidate_features[:n_features]):\n            print(f\"\\n[{i+1}/{n_features}] Testing feature: {feature}\")\n            \n            # Create bucketed feature\n            bucket_col = f\"{feature}_bucket\"\n            self.train_df[bucket_col] = self.bucket_transformer.transform(self.train_df, feature)\n            self.test_df[bucket_col] = self.bucket_transformer.transform(self.test_df, feature)\n            \n            # Test with each model type\n            test_features = self.baseline_features + [bucket_col]\n            \n            for model_type in [\"XGBoost\", \"LightGBM\"]:  # Skip NNs for speed in exploration\n                score, _, _ = train_model_cv(self.train_df, y, model_type, test_features)\n                improvement = score - self.baseline_score\n                \n                print(f\"   {model_type}: {score:.6f} (improvement: {improvement:+.6f})\")\n                \n                self.tracker.record_feature_test(bucket_col, score, model_type, improvement)\n                feature_scores.append((feature, score, improvement))\n            \n            # Clean up\n            self.train_df.drop(columns=[bucket_col], inplace=True)\n            self.test_df.drop(columns=[bucket_col], inplace=True)\n            \n        # Print interim summary\n        self.tracker.print_summary()\n        \n        return feature_scores\n    \n    def apply_vae_reduction(self, n_top_features: int = 10, latent_dim: int = 5):\n        \"\"\"Apply VAE to top bucketed features\"\"\"\n        print(f\"\\n🧬 APPLYING VAE DIMENSIONALITY REDUCTION\")\n        print(f\"   Top features: {n_top_features}\")\n        print(f\"   Latent dimensions: {latent_dim}\")\n        print(\"=\"*60)\n        \n        # Get top performing features\n        top_features = self.tracker.get_top_features(n_top_features)\n        \n        if not top_features:\n            print(\"No features to reduce. Skipping VAE.\")\n            return\n            \n        # Create bucketed versions of top features\n        bucketed_features = []\n        for feature in top_features:\n            base_feature = feature.replace('_bucket', '')\n            if base_feature in self.candidate_features:\n                bucket_col = f\"{base_feature}_bucket\"\n                self.train_df[bucket_col] = self.bucket_transformer.transform(self.train_df, base_feature)\n                self.test_df[bucket_col] = self.bucket_transformer.transform(self.test_df, base_feature)\n                bucketed_features.append(bucket_col)\n        \n        if not bucketed_features:\n            print(\"No valid features for VAE. Skipping.\")\n            return\n            \n        print(f\"Created {len(bucketed_features)} bucketed features for VAE\")\n        \n        # Prepare data for VAE\n        X_vae_train = self.train_df[bucketed_features].values.astype(np.float32)\n        X_vae_test = self.test_df[bucketed_features].values.astype(np.float32)\n        \n        # Normalize for VAE\n        scaler = StandardScaler()\n        X_vae_train_scaled = scaler.fit_transform(X_vae_train)\n        X_vae_test_scaled = scaler.transform(X_vae_test)\n        \n        # Train VAE\n        print(\"Training VAE...\")\n        vae = VAE(input_dim=len(bucketed_features), latent_dim=latent_dim)\n        vae.compile(optimizer='adam', loss='mse')\n        \n        vae.fit(\n            X_vae_train_scaled, X_vae_train_scaled,\n            epochs=50,\n            batch_size=256,\n            validation_split=0.1,\n            verbose=0\n        )\n        \n        # Get latent features\n        latent_train = vae.get_latent_features(X_vae_train_scaled).numpy()\n        latent_test = vae.get_latent_features(X_vae_test_scaled).numpy()\n        \n        # Add latent features to dataframes\n        for i in range(latent_dim):\n            col_name = f'vae_latent_{i}'\n            self.train_df[col_name] = latent_train[:, i]\n            self.test_df[col_name] = latent_test[:, i]\n        \n        vae_features = [f'vae_latent_{i}' for i in range(latent_dim)]\n        \n        # Test performance with VAE features\n        print(\"\\nTesting models with VAE features...\")\n        y = self.train_df[Config.LABEL_COLUMN]\n        \n        test_features = self.baseline_features + vae_features\n        \n        for model_type in [\"XGBoost\", \"LightGBM\", \"Balanced NN\", \"Wide NN\"]:\n            score, _, _ = train_model_cv(self.train_df, y, model_type, test_features)\n            improvement = score - self.baseline_score\n            \n            print(f\"   {model_type} with VAE: {score:.6f} (improvement: {improvement:+.6f})\")\n            \n            self.tracker.record_feature_test(\"VAE_features\", score, model_type, improvement)\n        \n        # Clean up bucketed features but keep VAE features for final model\n        for col in bucketed_features:\n            self.train_df.drop(columns=[col], inplace=True)\n            self.test_df.drop(columns=[col], inplace=True)\n            \n        return vae_features\n\n# ===== Main Pipeline =====\ndef main():\n    \"\"\"Main execution pipeline\"\"\"\n    print(\"🚀 STREAMLINED CRYPTO PREDICTION PIPELINE\")\n    print(\"=\"*80)\n    print(\"📋 PIPELINE OVERVIEW:\")\n    print(\"   1. Load data with core + engineered features\")\n    print(\"   2. Establish baseline performance\")\n    print(\"   3. Explore single bucketed feature additions\")\n    print(\"   4. Apply VAE to top features\")\n    print(\"   5. Generate final predictions\")\n    print(\"=\"*80)\n    \n    # Load data\n    print(\"\\n📁 LOADING DATA...\")\n    train_df = pd.read_parquet(Config.TRAIN_PATH)\n    test_df = pd.read_parquet(Config.TEST_PATH)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    \n    # Apply feature engineering\n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    print(f\"   Train shape: {train_df.shape}\")\n    print(f\"   Test shape: {test_df.shape}\")\n    \n    # Initialize feature explorer\n    explorer = FeatureExplorer(train_df, test_df)\n    \n    # Step 1: Establish baseline\n    explorer.establish_baseline()\n    \n    # Step 2: Explore single features\n    feature_scores = explorer.explore_single_features(n_features=5)\n    \n    # Step 3: Apply VAE to top features\n    vae_features = explorer.apply_vae_reduction(n_top_features=10, latent_dim=5)\n    \n    # Step 4: Final performance summary\n    print(\"\\n\" + \"=\"*80)\n    print(\"🏁 FINAL PERFORMANCE SUMMARY\")\n    print(\"=\"*80)\n    \n    explorer.tracker.print_summary()\n    \n    # Generate recommendations\n    print(\"\\n💡 RECOMMENDATIONS\")\n    print(\"=\"*60)\n    \n    top_features = explorer.tracker.get_top_features(3)\n    if top_features:\n        print(f\"1. Top performing bucketed features:\")\n        for i, feat in enumerate(top_features, 1):\n            print(f\"   {i}. {feat}\")\n    \n    if vae_features:\n        print(f\"\\n2. VAE reduced {len(top_features)} features to {len(vae_features)} latent dimensions\")\n        print(\"   Consider using VAE features for production if improvement is significant\")\n    \n    print(\"\\n3. Next steps:\")\n    print(\"   • Test ensemble methods combining best models\")\n    print(\"   • Explore interaction features between top buckets\")\n    print(\"   • Fine-tune hyperparameters for best model configuration\")\n    print(\"   • Consider feature selection within buckets\")\n    \n    # Create final submission with best configuration\n    print(\"\\n📝 CREATING FINAL SUBMISSION...\")\n    \n    # Use best performing configuration\n    best_result = max(explorer.tracker.results, key=lambda x: x['score'])\n    print(f\"Using configuration: {best_result['model_type']} with {best_result['feature']}\")\n    \n    # Train final model on full data\n    if vae_features and \"VAE\" in best_result['feature']:\n        final_features = explorer.baseline_features + vae_features\n    else:\n        final_features = explorer.baseline_features\n    \n    y = train_df[Config.LABEL_COLUMN]\n    \n    if best_result['model_type'] == \"XGBoost\":\n        final_model = XGBRegressor(**XGB_PARAMS)\n        final_model.fit(train_df[final_features].values, y)\n        predictions = final_model.predict(test_df[final_features].values)\n    elif best_result['model_type'] == \"LightGBM\":\n        final_model = LGBMRegressor(**LGBM_PARAMS)\n        final_model.fit(train_df[final_features].values, y)\n        predictions = final_model.predict(test_df[final_features].values)\n    \n    # Create submission\n    submission_df['prediction'] = predictions\n    submission_df.to_csv('submission_optimized.csv', index=False)\n    print(\"✅ Submission saved to 'submission_optimized.csv'\")\n    \n    print(\"\\n🎯 PIPELINE COMPLETED SUCCESSFULLY!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}