{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"a6a5a4fa-e795-43ec-949a-13408ee913e3","_cell_guid":"54444a80-4a5d-499d-8021-ad44b932453f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-07-24T20:50:49.242596Z","iopub.execute_input":"2025-07-24T20:50:49.242858Z","iopub.status.idle":"2025-07-24T20:50:49.507087Z","shell.execute_reply.started":"2025-07-24T20:50:49.242837Z","shell.execute_reply":"2025-07-24T20:50:49.506391Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================\n# INSTALLATIONS\n# ============================================\n!pip install -q prophet\n!pip install -q koolbox\n!pip install -q scikit-learn==1.5.2\n!pip install -q autogluon\n!pip install -q flaml[automl]\n!pip install -q mljar-supervised\n!pip install -q h2o\n!pip install -q optuna\n!pip install -q lightgbm\n!pip install -q xgboost\n!pip install -q catboost\n\n# ============================================\n# IMPORTS\n# ============================================\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom prophet import Prophet\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.ensemble import HistGradientBoostingRegressor, RandomForestRegressor, ExtraTreesRegressor\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, RidgeCV\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.decomposition import PCA, FastICA\nfrom sklearn.feature_selection import SelectFromModel\nfrom lightgbm import LGBMRegressor, LGBMClassifier\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom scipy.stats import pearsonr\nfrom sklearn.base import clone, BaseEstimator, RegressorMixin\nfrom koolbox import Trainer\nimport joblib\nimport gc\nimport os\n\n# ============================================\n# CONFIGURATION\n# ============================================\nclass CFG:\n    train_path = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    test_path = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    sample_sub_path = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    \n    target = \"label\"\n    n_folds = 5\n    seed = 42\n    \n    # Feature list\n    X_FEATURES = ['X363', 'X321', 'X405', 'X730', 'X523', 'X756', 'X589', 'X462', 'X779',\n                  'X25', 'X532', 'X520', 'X329', 'X383', 'X751', 'X535', 'X639', 'X596', 'X761',\n                  \"X752\", \"X287\", \"X298\", \"X759\", \"X302\", \"X55\", \"X56\", \"X52\", \"X303\", \"X51\",\n                  \"X598\", \"X385\", \"X603\", \"X674\", \"X415\", \"X345\", \"X174\", \"X178\", \"X168\", \"X612\",\n                  \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\"]\n\n# ============================================\n# UTILITY FUNCTIONS\n# ============================================\ndef _pearsonr(y_true, y_pred):\n    return pearsonr(y_true, y_pred)[0]\n\ndef reduce_mem_usage(dataframe, dataset):    \n    print(f'Reducing memory usage for: {dataset}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        \n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                dataframe[col] = dataframe[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                dataframe[col] = dataframe[col].astype(np.float32)\n            else:\n                dataframe[col] = dataframe[col].astype(np.float64)\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print(f'--- Memory usage before: {initial_mem_usage:.2f} MB')\n    print(f'--- Memory usage after: {final_mem_usage:.2f} MB')\n    print(f'--- Decreased memory usage by {100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}%\\n')\n\n    return dataframe\n\n# ============================================\n# FEATURE ENGINEERING\n# ============================================\ndef feature_engineering(df):\n    # Original features\n    df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n    \n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n    df['log_volume'] = np.log1p(df['volume'])\n    \n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-10)\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-10)\n    \n    # New microstructure features\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n    df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n    \n    # Liquidity Depth Measures\n    df['total_depth'] = df['bid_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['log_depth'] = np.log1p(df['total_depth'])\n    \n    # Order Flow Toxicity Proxies\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity'] = np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    \n    # Market Activity Indicators\n    df['volume_depth_ratio'] = df['volume'] / (df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['bid_qty'])\n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    \n    # Fill NaN with median\n    for col in df.columns:\n        if df[col].isna().any():\n            median_val = df[col].median()\n            df[col] = df[col].fillna(median_val if not pd.isna(median_val) else 0)\n    \n    return df\n\n# ============================================\n# OUTLIER DETECTION WITH PROPHET\n# ============================================\ndef prophet_outlier_detection(df, feature_col, timestamp_col='__index_level_0__'):\n    \"\"\"Simple Prophet-based outlier detection\"\"\"\n    print(f\"Detecting outliers in {feature_col}...\")\n    \n    # Prepare data for Prophet\n    prophet_df = pd.DataFrame({\n        'ds': df[timestamp_col],\n        'y': df[feature_col]\n    })\n    \n    # Remove obvious bad values\n    prophet_df = prophet_df[np.isfinite(prophet_df['y'])]\n    \n    # Resample to hourly for efficiency\n    prophet_hourly = prophet_df.set_index('ds').resample('1H').mean().reset_index()\n    prophet_hourly = prophet_hourly.dropna()\n    \n    # Fit Prophet model\n    model = Prophet(\n        changepoint_prior_scale=0.05,\n        interval_width=0.95,\n        yearly_seasonality=False,\n        weekly_seasonality=True,\n        daily_seasonality=True\n    )\n    \n    model.fit(prophet_hourly)\n    \n    # Generate predictions\n    forecast = model.predict(prophet_hourly)\n    \n    # Identify outliers\n    residuals = prophet_hourly['y'] - forecast['yhat']\n    residual_std = residuals.std()\n    outliers_mask = np.abs(residuals) > 3 * residual_std\n    \n    outlier_pct = outliers_mask.sum() / len(prophet_hourly) * 100\n    print(f\"Found {outliers_mask.sum()} outliers ({outlier_pct:.2f}%)\")\n    \n    return outliers_mask, residual_std\n\n# ============================================\n# ADVANCED MODELS\n# ============================================\nclass NoiseAwareFeatureCompressor:\n    \"\"\"Advanced feature compression with noise awareness\"\"\"\n    def __init__(self, n_components=30, noise_threshold=0.1):\n        self.n_components = n_components\n        self.noise_threshold = noise_threshold\n        self.pca = None\n        self.scaler = StandardScaler()\n        self.noise_mask = None\n        \n    def fit(self, X, y=None):\n        X_scaled = self.scaler.fit_transform(X)\n        \n        # Estimate noise level per feature\n        noise_levels = []\n        for col in range(X_scaled.shape[1]):\n            diff = np.diff(X_scaled[:, col])\n            noise_estimate = np.std(diff) / np.sqrt(2)\n            noise_levels.append(noise_estimate)\n        \n        noise_levels = np.array(noise_levels)\n        self.noise_mask = noise_levels < np.percentile(noise_levels, 100 * (1 - self.noise_threshold))\n        \n        # Apply PCA on low-noise features\n        X_clean = X_scaled[:, self.noise_mask]\n        self.pca = PCA(n_components=min(self.n_components, X_clean.shape[1]))\n        self.pca.fit(X_clean)\n        \n        return self\n    \n    def transform(self, X):\n        X_scaled = self.scaler.transform(X)\n        X_clean = X_scaled[:, self.noise_mask]\n        pca_features = self.pca.transform(X_clean)\n        high_noise_features = X_scaled[:, ~self.noise_mask]\n        \n        return np.hstack([pca_features, high_noise_features])\n    \n    def fit_transform(self, X, y=None):\n        return self.fit(X, y).transform(X)\n\n# ============================================\n# AUTOML FUNCTIONS\n# ============================================\ndef train_autogluon(X_train, y_train, X_test):\n    \"\"\"AutoGluon with hierarchical ensemble\"\"\"\n    try:\n        from autogluon.tabular import TabularPredictor\n        \n        train_data = X_train.copy()\n        train_data['label'] = y_train\n        \n        predictor = TabularPredictor(\n            label='label',\n            problem_type='regression',\n            eval_metric=lambda y_true, y_pred: pearsonr(y_true, y_pred)[0],\n            path='autogluon_models'\n        )\n        \n        predictor.fit(\n            train_data=train_data,\n            presets='best_quality',\n            ag_args_fit={\n                'num_bag_folds': 10,\n                'num_bag_sets': 3,\n                'num_stack_levels': 2,\n                'refit_full': True\n            },\n            time_limit=1800,\n            verbosity=1\n        )\n        \n        predictions = predictor.predict(X_test)\n        oof_predictions = predictor.get_oof_pred()\n        \n        return predictions, oof_predictions, predictor\n    except Exception as e:\n        print(f\"AutoGluon failed: {e}\")\n        return None, None, None\n\ndef train_flaml(X_train, y_train, X_test):\n    \"\"\"FLAML AutoML\"\"\"\n    try:\n        from flaml import AutoML\n        \n        automl = AutoML()\n        settings = {\n            \"time_budget\": 1800,\n            \"metric\": lambda y_true, y_pred: pearsonr(y_true, y_pred)[0],\n            \"task\": \"regression\",\n            \"n_splits\": 5,\n            \"eval_method\": \"cv\",\n            \"seed\": 42,\n            \"learner_selector\": \"tournament\",\n            \"estimator_list\": ['lgbm', 'xgboost', 'catboost', 'rf', 'extra_tree', 'lrl1', 'lrl2']\n        }\n        \n        automl.fit(X_train, y_train, **settings)\n        predictions = automl.predict(X_test)\n        \n        # Get OOF predictions\n        kf = KFold(n_splits=5, shuffle=False)\n        oof_predictions = np.zeros(len(X_train))\n        \n        for train_idx, val_idx in kf.split(X_train):\n            X_fold_train, X_fold_val = X_train.iloc[train_idx], X_train.iloc[val_idx]\n            y_fold_train = y_train.iloc[train_idx]\n            \n            model = automl.model.estimator.__class__(**automl.model.estimator.get_params())\n            model.fit(X_fold_train, y_fold_train)\n            oof_predictions[val_idx] = model.predict(X_fold_val)\n        \n        return predictions, oof_predictions, automl\n    except Exception as e:\n        print(f\"FLAML failed: {e}\")\n        return None, None, None\n\n# ============================================\n# MAIN PIPELINE\n# ============================================\ndef main():\n    print(\"Starting DRW Crypto Market Prediction Pipeline...\")\n    \n    # Load data\n    print(\"\\n1. Loading data...\")\n    train = pd.read_parquet(CFG.train_path).reset_index(drop=True)\n    test = pd.read_parquet(CFG.test_path).reset_index(drop=True)\n    \n    # Select features\n    selected_columns = CFG.X_FEATURES + [\"volume\"]\n    train = train[selected_columns + [CFG.target]]\n    test = test[selected_columns]\n    \n    # Add timestamp if missing\n    if '__index_level_0__' not in train.columns:\n        train['__index_level_0__'] = pd.date_range('2023-03-01', periods=len(train), freq='T')\n    if '__index_level_0__' not in test.columns:\n        test['__index_level_0__'] = pd.date_range('2024-03-01', periods=len(test), freq='T')\n    \n    # Apply feature engineering\n    print(\"\\n2. Feature Engineering...\")\n    train = feature_engineering(train)\n    test = feature_engineering(test)\n    \n    # Remove base features\n    to_remove = [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"__index_level_0__\"]\n    train = train.drop(columns=[col for col in to_remove if col in train.columns])\n    test = test.drop(columns=[col for col in to_remove if col in test.columns])\n    \n    # Reduce memory\n    train = reduce_mem_usage(train, \"train\")\n    test = reduce_mem_usage(test, \"test\")\n    \n    # Prepare data\n    X = train.drop(CFG.target, axis=1)\n    y = train[CFG.target]\n    X_test = test\n    \n    # Store results\n    all_oof_preds = {}\n    all_test_preds = {}\n    all_scores = {}\n    \n    # ============================================\n    # FIRST LEVEL MODELS\n    # ============================================\n    print(\"\\n3. Training First Level Models...\")\n    \n    # Model parameters\n    lgbm_params = {\n        \"boosting_type\": \"gbdt\",\n        \"colsample_bytree\": 0.5625888953382505,\n        \"learning_rate\": 0.029312951475451557,\n        \"min_child_samples\": 63,\n        \"min_child_weight\": 0.11456572852335424,\n        \"n_estimators\": 126,\n        \"n_jobs\": -1,\n        \"num_leaves\": 37,\n        \"random_state\": 42,\n        \"reg_alpha\": 85.2476527854083,\n        \"reg_lambda\": 99.38305361388907,\n        \"subsample\": 0.450669817684892,\n        \"verbose\": -1\n    }\n    \n    xgb_params = {\n        \"colsample_bylevel\": 0.4778015829774066,\n        \"colsample_bynode\": 0.362764358742407,\n        \"colsample_bytree\": 0.7107423488010493,\n        \"gamma\": 1.7094857725240398,\n        \"learning_rate\": 0.02213323588455387,\n        \"max_depth\": 20,\n        \"max_leaves\": 12,\n        \"min_child_weight\": 16,\n        \"n_estimators\": 1667,\n        \"n_jobs\": -1,\n        \"random_state\": 42,\n        \"reg_alpha\": 39.352415706891264,\n        \"reg_lambda\": 75.44843704068275,\n        \"subsample\": 0.06566669853471274,\n        \"verbosity\": 0\n    }\n    \n    # Train models\n    models = {\n        'Lasso': Lasso(alpha=0.0005, max_iter=10000),\n        'Ridge': Ridge(alpha=1.0),\n        'ElasticNet': ElasticNet(alpha=0.001, l1_ratio=0.5),\n        'LightGBM': LGBMRegressor(**lgbm_params),\n        'XGBoost': XGBRegressor(**xgb_params),\n        'RandomForest': RandomForestRegressor(n_estimators=200, max_depth=10, min_samples_leaf=20, random_state=42),\n        'ExtraTrees': ExtraTreesRegressor(n_estimators=200, max_depth=8, min_samples_leaf=30, random_state=42)\n    }\n    \n    for name, model in models.items():\n        print(f\"\\nTraining {name}...\")\n        try:\n            trainer = Trainer(\n                model,\n                cv=KFold(n_splits=5, shuffle=False),\n                metric=_pearsonr,\n                task=\"regression\",\n                metric_precision=6\n            )\n            trainer.fit(X, y)\n            \n            all_scores[name] = trainer.fold_scores\n            all_oof_preds[name] = trainer.oof_preds\n            all_test_preds[name] = trainer.predict(X_test)\n            \n            print(f\"{name} - Mean Score: {np.mean(trainer.fold_scores):.6f}\")\n        except Exception as e:\n            print(f\"{name} failed: {e}\")\n    \n    # ============================================\n    # AUTOML MODELS\n    # ============================================\n    print(\"\\n4. Training AutoML Models...\")\n    \n    # AutoGluon\n    print(\"\\nTraining AutoGluon...\")\n    ag_pred, ag_oof, ag_model = train_autogluon(X, y, X_test)\n    if ag_pred is not None:\n        all_test_preds['AutoGluon'] = ag_pred\n        all_oof_preds['AutoGluon'] = ag_oof\n        train_score = _pearsonr(y, ag_oof)\n        all_scores['AutoGluon'] = [train_score] * 5\n        print(f\"AutoGluon - Train Score: {train_score:.6f}\")\n    \n    # FLAML\n    print(\"\\nTraining FLAML...\")\n    flaml_pred, flaml_oof, flaml_model = train_flaml(X, y, X_test)\n    if flaml_pred is not None:\n        all_test_preds['FLAML'] = flaml_pred\n        all_oof_preds['FLAML'] = flaml_oof\n        train_score = _pearsonr(y, flaml_oof)\n        all_scores['FLAML'] = [train_score] * 5\n        print(f\"FLAML - Train Score: {train_score:.6f}\")\n    \n    # ============================================\n    # ADVANCED TECHNIQUES\n    # ============================================\n    print(\"\\n5. Applying Advanced Techniques...\")\n    \n    # Noise-Aware Feature Compression\n    print(\"\\nApplying Noise-Aware Compression...\")\n    compressor = NoiseAwareFeatureCompressor(n_components=30)\n    X_compressed = pd.DataFrame(compressor.fit_transform(X))\n    X_test_compressed = pd.DataFrame(compressor.transform(X_test))\n    \n    # Train model on compressed features\n    lgbm_compressed = LGBMRegressor(\n        n_estimators=200,\n        learning_rate=0.03,\n        num_leaves=31,\n        subsample=0.7,\n        colsample_bytree=0.7,\n        reg_alpha=20,\n        reg_lambda=20,\n        min_child_samples=50,\n        random_state=42,\n        verbose=-1\n    )\n    \n    trainer_compressed = Trainer(\n        lgbm_compressed,\n        cv=KFold(n_splits=5, shuffle=False),\n        metric=_pearsonr,\n        task=\"regression\",\n        metric_precision=6\n    )\n    trainer_compressed.fit(X_compressed, y)\n    \n    all_scores['LGBM_Compressed'] = trainer_compressed.fold_scores\n    all_oof_preds['LGBM_Compressed'] = trainer_compressed.oof_preds\n    all_test_preds['LGBM_Compressed'] = trainer_compressed.predict(X_test_compressed)\n    print(f\"LGBM_Compressed - Mean Score: {np.mean(trainer_compressed.fold_scores):.6f}\")\n    \n    # ============================================\n    # FINAL ENSEMBLE\n    # ============================================\n    print(\"\\n6. Creating Final Ensemble...\")\n    \n    # Create ensemble features\n    X_ensemble = pd.DataFrame(all_oof_preds)\n    X_test_ensemble = pd.DataFrame(all_test_preds)\n    \n    # Train meta-model\n    meta_model = RidgeCV(alphas=[0.001, 0.01, 0.1, 1.0, 10.0, 100.0], cv=5)\n    meta_model.fit(X_ensemble, y)\n    \n    # Get final predictions\n    final_predictions = meta_model.predict(X_test_ensemble)\n    \n    # Print ensemble weights\n    print(\"\\nEnsemble Weights:\")\n    for name, weight in zip(X_ensemble.columns, meta_model.coef_):\n        print(f\"{name}: {weight:.4f}\")\n    \n    # Calculate ensemble OOF score\n    ensemble_oof = meta_model.predict(X_ensemble)\n    ensemble_score = _pearsonr(y, ensemble_oof)\n    print(f\"\\nEnsemble Train Score: {ensemble_score:.6f}\")\n    \n    # ============================================\n    # SAVE RESULTS\n    # ============================================\n    print(\"\\n7. Saving Results...\")\n    \n    # Save predictions\n    sub = pd.read_csv(CFG.sample_sub_path)\n    sub[\"prediction\"] = final_predictions\n    sub.to_csv(\"submission.csv\", index=False)\n    print(\"Submission saved to submission.csv\")\n    \n    # Save OOF predictions\n    joblib.dump(all_oof_preds, \"all_oof_preds.pkl\")\n    joblib.dump(all_test_preds, \"all_test_preds.pkl\")\n    \n    # Plot results\n    scores_df = pd.DataFrame(all_scores)\n    mean_scores = scores_df.mean().sort_values(ascending=False)\n    \n    plt.figure(figsize=(12, 8))\n    plt.subplot(2, 1, 1)\n    scores_df.boxplot(vert=False)\n    plt.title(\"Model Scores Distribution\")\n    plt.xlabel(\"Pearson Correlation\")\n    \n    plt.subplot(2, 1, 2)\n    mean_scores.plot(kind='barh')\n    plt.title(\"Average Model Scores\")\n    plt.xlabel(\"Pearson Correlation\")\n    \n    plt.tight_layout()\n    plt.savefig('model_performance.png')\n    plt.show()\n    \n    print(\"\\nPipeline completed successfully!\")\n    \n    return final_predictions, ensemble_score\n\n# ============================================\n# RUN MAIN PIPELINE\n# ============================================\nif __name__ == \"__main__\":\n    final_predictions, ensemble_score = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}