{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.preprocessing import RobustScaler, QuantileTransformer\nfrom sklearn.feature_selection import SelectKBest, f_regression\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr, rankdata\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ============================================================================\n# BASELINE CONFIGURATION (EXACT COPY)\n# ============================================================================\n\ndef feature_engineering_baseline(df):\n    \"\"\"Original feature engineering function\"\"\"\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    return df \n\nclass ConfigBaseline:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n\nXGB_PARAMS_BASELINE = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": ConfigBaseline.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\nLEARNERS_BASELINE = [\n    {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS_BASELINE},\n]\n\n# ============================================================================\n# ENHANCED CONFIGURATION WITH ANTI-OVERFITTING\n# ============================================================================\n\ndef safe_divide(a, b, default=0):\n    \"\"\"Safe division with handling of inf/nan values\"\"\"\n    result = np.divide(a, b, out=np.full_like(a, default, dtype=float), where=(b != 0))\n    return np.nan_to_num(result, nan=default, posinf=default, neginf=default)\n\ndef create_microstructure_features(df):\n    \"\"\"Create 5 new microstructure features based on market data\"\"\"\n    features = df.copy()\n    \n    # 1. Order Flow Imbalance\n    features['order_flow_imbalance'] = safe_divide(\n        features['bid_qty'] - features['ask_qty'],\n        features['bid_qty'] + features['ask_qty'] + 1e-8\n    )\n    \n    # 2. Trade Flow Imbalance  \n    features['trade_flow_imbalance'] = safe_divide(\n        features['buy_qty'] - features['sell_qty'],\n        features['buy_qty'] + features['sell_qty'] + 1e-8\n    )\n    \n    # 3. Market Depth Ratio\n    features['market_depth_ratio'] = safe_divide(\n        features['bid_qty'],\n        features['ask_qty'] + 1e-8\n    )\n    \n    # 4. Trade Efficiency\n    features['trade_efficiency'] = safe_divide(\n        features['volume'],\n        features['buy_qty'] + features['sell_qty'] + 1e-8\n    )\n    \n    # 5. Liquidity Concentration\n    features['liquidity_concentration'] = safe_divide(\n        features['bid_qty'] + features['ask_qty'],\n        features['volume'] + 1e-8\n    )\n    \n    return features\n\ndef create_proprietary_rank_features(df, n_features=5):\n    \"\"\"Create rank/bucket features from proprietary X variables\"\"\"\n    features = df.copy()\n    \n    # Get X columns with highest variance (most informative)\n    x_cols = [col for col in df.columns if col.startswith('X')]\n    if len(x_cols) == 0:\n        return features\n    \n    # Calculate variance for each X column\n    variances = []\n    for col in x_cols:\n        var_val = np.var(df[col].fillna(0))\n        variances.append((col, var_val))\n    \n    # Sort by variance and take top features\n    variances.sort(key=lambda x: x[1], reverse=True)\n    top_x_features = [col for col, _ in variances[:n_features]]\n    \n    for i, col in enumerate(top_x_features):\n        if col in df.columns:\n            # 1. Rank transformation\n            values = df[col].fillna(df[col].median())\n            features[f'{col}_rank'] = rankdata(values, method='average') / len(values)\n            \n            # 2. Quantile buckets (10 buckets)\n            features[f'{col}_bucket'] = pd.qcut(\n                values, \n                q=10, \n                labels=False, \n                duplicates='drop'\n            ).fillna(5)  # median bucket for NaN\n    \n    return features\n\ndef feature_engineering_enhanced(df):\n    \"\"\"Enhanced feature engineering with microstructure and rank features\"\"\"\n    # Start with baseline features\n    df = feature_engineering_baseline(df)\n    \n    # Add microstructure features\n    df = create_microstructure_features(df)\n    \n    # Add proprietary rank features\n    df = create_proprietary_rank_features(df, n_features=5)\n    \n    # Additional safety for inf/nan values\n    df = df.replace([np.inf, -np.inf], np.nan)\n    \n    # Use more sophisticated imputation\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n    for col in numeric_cols:\n        median_val = df[col].median()\n        df[col] = df[col].fillna(median_val)\n    \n    return df\n\nclass ConfigEnhanced:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    # Start with baseline features\n    FEATURES_BASE = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\", \"bid_qty\", \"ask_qty\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    RANDOM_STATE = 42\n\n# Anti-overfitting XGB parameters (more conservative)\nXGB_PARAMS_CONSERVATIVE = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.6,\n    \"colsample_bynode\": 0.6,\n    \"colsample_bytree\": 0.8,\n    \"gamma\": 2.0,\n    \"learning_rate\": 0.01,  # Lower learning rate\n    \"max_depth\": 8,  # Reduced depth\n    \"max_leaves\": 8,  # Reduced leaves\n    \"min_child_weight\": 20,  # Higher min child weight\n    \"n_estimators\": 2000,\n    \"subsample\": 0.7,\n    \"reg_alpha\": 50,  # Higher regularization\n    \"reg_lambda\": 100,  # Higher regularization\n    \"verbosity\": 0,\n    \"random_state\": ConfigEnhanced.RANDOM_STATE,\n    \"n_jobs\": -1,\n    \"early_stopping_rounds\": 100\n}\n\nLGBM_PARAMS_CONSERVATIVE = {\n    \"objective\": \"regression\",\n    \"metric\": \"rmse\",\n    \"boosting_type\": \"gbdt\",\n    \"num_leaves\": 15,  # Conservative\n    \"learning_rate\": 0.01,\n    \"feature_fraction\": 0.7,\n    \"bagging_fraction\": 0.7,\n    \"bagging_freq\": 5,\n    \"min_child_samples\": 25,\n    \"reg_alpha\": 50,\n    \"reg_lambda\": 100,\n    \"n_estimators\": 2000,\n    \"random_state\": ConfigEnhanced.RANDOM_STATE,\n    \"verbosity\": -1,\n    \"n_jobs\": -1\n}\n\n# ============================================================================\n# WEIGHT STRATEGIES\n# ============================================================================\n\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    \"\"\"Original time decay weights\"\"\"\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef create_exponential_weights(n: int, decay: float = 0.95) -> np.ndarray:\n    \"\"\"Exponential decay weights (stronger recency bias)\"\"\"\n    positions = np.arange(n)\n    weights = decay ** (n - 1 - positions)\n    return weights * n / weights.sum()\n\ndef create_linear_weights(n: int) -> np.ndarray:\n    \"\"\"Linear increasing weights\"\"\"\n    weights = np.linspace(0.1, 1.0, n)\n    return weights * n / weights.sum()\n\ndef create_uniform_weights(n: int) -> np.ndarray:\n    \"\"\"Uniform weights\"\"\"\n    return np.ones(n) / n * n\n\nWEIGHT_STRATEGIES = {\n    'time_decay': create_time_decay_weights,\n    'exponential': create_exponential_weights, \n    'linear': create_linear_weights,\n    'uniform': create_uniform_weights\n}\n\n# ============================================================================\n# BASELINE FUNCTIONS (EXACT COPY)\n# ============================================================================\n\ndef load_data_baseline():\n    ConfigBaseline.FEATURES += [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\n    ConfigBaseline.FEATURES = list(set(ConfigBaseline.FEATURES))\n    \n    train_df = pd.read_parquet(ConfigBaseline.TRAIN_PATH, columns=ConfigBaseline.FEATURES + [ConfigBaseline.LABEL_COLUMN])\n    test_df = pd.read_parquet(ConfigBaseline.TEST_PATH, columns=ConfigBaseline.FEATURES)\n    submission_df = pd.read_csv(ConfigBaseline.SUBMISSION_PATH)\n\n    train_df = feature_engineering_baseline(train_df)\n    test_df = feature_engineering_baseline(test_df)\n    print(f\"Baseline - Loaded data - Train: {train_df.shape}, Test: {test_df.shape}\")\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\ndef get_model_slices(n_samples: int):\n    return [\n        {\"name\": \"full_data\", \"cutoff\": 0},\n        {\"name\": \"last_75pct\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_50pct\", \"cutoff\": int(0.50 * n_samples)},\n    ]\n\ndef train_and_evaluate_baseline(train_df, test_df):\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS_BASELINE\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS_BASELINE\n    }\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=ConfigBaseline.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Baseline Fold {fold}/{ConfigBaseline.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][ConfigBaseline.FEATURES]\n        y_valid = train_df.iloc[valid_idx][ConfigBaseline.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            subset = train_df.iloc[cutoff:].reset_index(drop=True)\n            rel_idx = train_idx[train_idx >= cutoff] - cutoff\n\n            X_train = subset.iloc[rel_idx][ConfigBaseline.FEATURES]\n            y_train = subset.iloc[rel_idx][ConfigBaseline.LABEL_COLUMN]\n            sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n            for learner in LEARNERS_BASELINE:\n                model = learner[\"Estimator\"](**learner[\"params\"])\n                model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n                mask = valid_idx >= cutoff\n                if mask.any():\n                    idxs = valid_idx[mask]\n                    oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][ConfigBaseline.FEATURES])\n                if cutoff > 0 and (~mask).any():\n                    oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n\n                test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[ConfigBaseline.FEATURES])\n\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= ConfigBaseline.N_FOLDS\n\n    return oof_preds, test_preds\n\ndef ensemble_and_submit_baseline(train_df, oof_preds, test_preds, submission_df):\n    learner_ensembles = {}\n    for learner_name in oof_preds:\n        scores = {s: pearsonr(train_df[ConfigBaseline.LABEL_COLUMN], oof_preds[learner_name][s])[0]\n                  for s in oof_preds[learner_name]}\n        \n        oof_simple = np.mean(list(oof_preds[learner_name].values()), axis=0)\n        test_simple = np.mean(list(test_preds[learner_name].values()), axis=0)\n        score_simple = pearsonr(train_df[ConfigBaseline.LABEL_COLUMN], oof_simple)[0]\n\n        print(f\"\\nBaseline {learner_name.upper()} Simple Ensemble Pearson: {score_simple:.4f}\")\n        \n        learner_ensembles[learner_name] = {\n            \"oof_simple\": oof_simple,\n            \"test_simple\": test_simple\n        }\n\n    final_oof = np.mean([le[\"oof_simple\"] for le in learner_ensembles.values()], axis=0)\n    final_test = np.mean([le[\"test_simple\"] for le in learner_ensembles.values()], axis=0)\n    final_score = pearsonr(train_df[ConfigBaseline.LABEL_COLUMN], final_oof)[0]\n\n    print(f\"\\nBaseline FINAL ensemble Pearson: {final_score:.4f}\")\n\n    submission_df[\"prediction\"] = final_test\n    submission_df.to_csv(\"submission_baseline.csv\", index=False)\n    print(\"Saved: submission_baseline.csv\")\n    \n    return final_score\n\n# ============================================================================\n# ENHANCED FUNCTIONS WITH ANTI-OVERFITTING\n# ============================================================================\n\ndef load_data_enhanced():\n    \"\"\"Load and prepare enhanced dataset\"\"\"\n    # Load all available columns first\n    train_df = pd.read_parquet(ConfigEnhanced.TRAIN_PATH)\n    test_df = pd.read_parquet(ConfigEnhanced.TEST_PATH)\n    submission_df = pd.read_csv(ConfigEnhanced.SUBMISSION_PATH)\n    \n    # Apply enhanced feature engineering\n    train_df = feature_engineering_enhanced(train_df)\n    test_df = feature_engineering_enhanced(test_df)\n    \n    # Get feature names (exclude label and timestamp)\n    feature_cols = [col for col in train_df.columns \n                   if col not in [ConfigEnhanced.LABEL_COLUMN, 'timestamp']]\n    \n    print(f\"Enhanced - Loaded data - Train: {train_df.shape}, Test: {test_df.shape}\")\n    print(f\"Total features available: {len(feature_cols)}\")\n    \n    return train_df, test_df, submission_df, feature_cols\n\ndef train_enhanced_model(model_name, train_df, test_df, feature_cols, n_folds=7, weight_strategy='time_decay'):\n    \"\"\"Train enhanced model with anti-overfitting measures\"\"\"\n    print(f\"\\n{'='*50}\")\n    print(f\"Training Enhanced Model: {model_name}\")\n    print(f\"Folds: {n_folds}, Weight Strategy: {weight_strategy}\")\n    print(f\"Features: {len(feature_cols)}\")\n    print(f\"{'='*50}\")\n    \n    n_samples = len(train_df)\n    \n    # Feature selection - select top k features to reduce overfitting\n    selector = SelectKBest(score_func=f_regression, k=min(100, len(feature_cols)))\n    X_full = train_df[feature_cols].fillna(0)\n    y_full = train_df[ConfigEnhanced.LABEL_COLUMN]\n    \n    X_selected = selector.fit_transform(X_full, y_full)\n    selected_features = [feature_cols[i] for i in selector.get_support(indices=True)]\n    print(f\"Selected {len(selected_features)} features after feature selection\")\n    \n    # Cross-validation setup\n    if n_folds <= 5:\n        cv = KFold(n_splits=n_folds, shuffle=False, random_state=ConfigEnhanced.RANDOM_STATE)\n    else:\n        # Use TimeSeriesSplit for more folds to better respect temporal structure\n        cv = TimeSeriesSplit(n_splits=n_folds)\n    \n    # Initialize predictions\n    oof_preds = np.zeros(n_samples)\n    test_preds = np.zeros(len(test_df))\n    feature_importance = np.zeros(len(selected_features))\n    \n    # Weight function\n    weight_func = WEIGHT_STRATEGIES[weight_strategy]\n    \n    fold_scores = []\n    overfitting_metrics = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(cv.split(train_df), start=1):\n        print(f\"\\n--- Enhanced Fold {fold}/{n_folds} ---\")\n        \n        # Prepare data\n        X_train = train_df.iloc[train_idx][selected_features]\n        y_train = train_df.iloc[train_idx][ConfigEnhanced.LABEL_COLUMN]\n        X_valid = train_df.iloc[valid_idx][selected_features]\n        y_valid = train_df.iloc[valid_idx][ConfigEnhanced.LABEL_COLUMN]\n        \n        # Create sample weights\n        sample_weights = weight_func(len(train_idx))\n        \n        # Model selection\n        if model_name == 'xgb_conservative':\n            model = XGBRegressor(**XGB_PARAMS_CONSERVATIVE)\n            model.fit(\n                X_train, y_train, \n                sample_weight=sample_weights,\n                eval_set=[(X_valid, y_valid)],\n                verbose=False\n            )\n        elif model_name == 'lgbm_conservative':\n            model = LGBMRegressor(**LGBM_PARAMS_CONSERVATIVE)\n            model.fit(\n                X_train, y_train,\n                sample_weight=sample_weights,\n                eval_set=[(X_valid, y_valid)],\n                callbacks=[lgb.early_stopping(stopping_rounds=100), lgb.log_evaluation(0)]\n            )\n        \n        # Predictions\n        oof_preds[valid_idx] = model.predict(X_valid)\n        test_preds += model.predict(test_df[selected_features]) / n_folds\n        \n        # Feature importance\n        if hasattr(model, 'feature_importances_'):\n            feature_importance += model.feature_importances_ / n_folds\n        \n        # Validation metrics\n        valid_score = pearsonr(y_valid, oof_preds[valid_idx])[0]\n        train_score = pearsonr(y_train, model.predict(X_train))[0]\n        overfitting = train_score - valid_score\n        \n        fold_scores.append(valid_score)\n        overfitting_metrics.append(overfitting)\n        \n        print(f\"  Fold {fold} - Valid Score: {valid_score:.4f}, Train Score: {train_score:.4f}, Overfitting: {overfitting:.4f}\")\n    \n    # Final metrics\n    final_oof_score = pearsonr(train_df[ConfigEnhanced.LABEL_COLUMN], oof_preds)[0]\n    avg_fold_score = np.mean(fold_scores)\n    std_fold_score = np.std(fold_scores)\n    avg_overfitting = np.mean(overfitting_metrics)\n    \n    print(f\"\\n{model_name.upper()} Results:\")\n    print(f\"  Final OOF Score: {final_oof_score:.4f}\")\n    print(f\"  Avg Fold Score: {avg_fold_score:.4f} ± {std_fold_score:.4f}\")\n    print(f\"  Avg Overfitting: {avg_overfitting:.4f}\")\n    \n    # Feature importance analysis\n    if len(feature_importance) > 0:\n        importance_df = pd.DataFrame({\n            'feature': selected_features,\n            'importance': feature_importance\n        }).sort_values('importance', ascending=False)\n        print(f\"\\nTop 10 Important Features:\")\n        print(importance_df.head(10).to_string(index=False))\n    \n    return {\n        'model_name': model_name,\n        'oof_preds': oof_preds,\n        'test_preds': test_preds,\n        'oof_score': final_oof_score,\n        'fold_scores': fold_scores,\n        'overfitting_metrics': overfitting_metrics,\n        'selected_features': selected_features,\n        'feature_importance': feature_importance\n    }\n\ndef run_enhanced_experiments(train_df, test_df, submission_df, feature_cols):\n    \"\"\"Run multiple enhanced experiments with different configurations\"\"\"\n    \n    experiments = [\n        {'model': 'xgb_conservative', 'folds': 5, 'weight': 'time_decay'},\n        {'model': 'xgb_conservative', 'folds': 7, 'weight': 'exponential'},\n        {'model': 'xgb_conservative', 'folds': 10, 'weight': 'linear'},\n        {'model': 'lgbm_conservative', 'folds': 7, 'weight': 'time_decay'},\n        {'model': 'lgbm_conservative', 'folds': 10, 'weight': 'exponential'},\n    ]\n    \n    results = []\n    \n    for i, exp in enumerate(experiments):\n        print(f\"\\n{'#'*60}\")\n        print(f\"EXPERIMENT {i+1}/{len(experiments)}\")\n        print(f\"{'#'*60}\")\n        \n        result = train_enhanced_model(\n            model_name=exp['model'],\n            train_df=train_df,\n            test_df=test_df,\n            feature_cols=feature_cols,\n            n_folds=exp['folds'],\n            weight_strategy=exp['weight']\n        )\n        \n        result['experiment_config'] = exp\n        results.append(result)\n        \n        # Save individual submission\n        submission_copy = submission_df.copy()\n        submission_copy[\"prediction\"] = result['test_preds']\n        filename = f\"submission_enhanced_{exp['model']}_f{exp['folds']}_{exp['weight']}.csv\"\n        submission_copy.to_csv(filename, index=False)\n        print(f\"Saved: {filename}\")\n    \n    return results\n\ndef create_ensemble_from_results(results, train_df, submission_df):\n    \"\"\"Create ensemble from multiple model results\"\"\"\n    print(f\"\\n{'='*50}\")\n    print(\"CREATING ENHANCED ENSEMBLE\")\n    print(f\"{'='*50}\")\n    \n    # Weight models by their OOF performance\n    weights = []\n    for result in results:\n        # Use max(0, score) to handle negative correlations\n        weight = max(0, result['oof_score'])\n        weights.append(weight)\n    \n    # Normalize weights\n    total_weight = sum(weights)\n    if total_weight > 0:\n        weights = [w / total_weight for w in weights]\n    else:\n        weights = [1/len(results)] * len(results)\n    \n    print(\"Model weights:\")\n    for i, (result, weight) in enumerate(zip(results, weights)):\n        exp = result['experiment_config']\n        print(f\"  {exp['model']}_f{exp['folds']}_{exp['weight']}: {weight:.4f} (OOF: {result['oof_score']:.4f})\")\n    \n    # Create weighted ensemble\n    ensemble_oof = np.zeros(len(train_df))\n    ensemble_test = np.zeros(len(submission_df))\n    \n    for result, weight in zip(results, weights):\n        ensemble_oof += weight * result['oof_preds']\n        ensemble_test += weight * result['test_preds']\n    \n    # Evaluate ensemble\n    ensemble_score = pearsonr(train_df[ConfigEnhanced.LABEL_COLUMN], ensemble_oof)[0]\n    print(f\"\\nEnsemble OOF Score: {ensemble_score:.4f}\")\n    \n    # Save ensemble submission\n    submission_ensemble = submission_df.copy()\n    submission_ensemble[\"prediction\"] = ensemble_test\n    submission_ensemble.to_csv(\"submission_enhanced_ensemble.csv\", index=False)\n    print(\"Saved: submission_enhanced_ensemble.csv\")\n    \n    return ensemble_score\n\ndef analyze_overfitting(results):\n    \"\"\"Analyze overfitting patterns across experiments\"\"\"\n    print(f\"\\n{'='*50}\")\n    print(\"OVERFITTING ANALYSIS\")\n    print(f\"{'='*50}\")\n    \n    analysis_df = []\n    for result in results:\n        exp = result['experiment_config']\n        row = {\n            'model': exp['model'],\n            'folds': exp['folds'],\n            'weight_strategy': exp['weight'],\n            'oof_score': result['oof_score'],\n            'avg_fold_score': np.mean(result['fold_scores']),\n            'std_fold_score': np.std(result['fold_scores']),\n            'avg_overfitting': np.mean(result['overfitting_metrics']),\n            'max_overfitting': np.max(result['overfitting_metrics']),\n            'stability': 1.0 / (1.0 + np.std(result['fold_scores']))  # Higher is more stable\n        }\n        analysis_df.append(row)\n    \n    analysis_df = pd.DataFrame(analysis_df)\n    print(\"\\nOverfitting Analysis Summary:\")\n    print(analysis_df.round(4).to_string(index=False))\n    \n    # Save analysis\n    analysis_df.to_csv(\"overfitting_analysis.csv\", index=False)\n    print(\"\\nSaved: overfitting_analysis.csv\")\n    \n    return analysis_df\n\n# ============================================================================\n# MAIN EXECUTION\n# ============================================================================\n\ndef main():\n    print(\"DRW Crypto Market Prediction - Baseline + Enhanced with Anti-Overfitting\")\n    print(\"=\"*80)\n    \n    # ========================================\n    # STEP 1: RUN BASELINE (EXACT COPY)\n    # ========================================\n    print(\"\\n\" + \"=\"*40)\n    print(\"RUNNING BASELINE MODEL\")\n    print(\"=\"*40)\n    \n    train_baseline, test_baseline, submission_baseline = load_data_baseline()\n    oof_baseline, test_baseline_preds = train_and_evaluate_baseline(train_baseline, test_baseline)\n    baseline_score = ensemble_and_submit_baseline(train_baseline, oof_baseline, test_baseline_preds, submission_baseline)\n    \n    # ========================================\n    # STEP 2: RUN ENHANCED MODELS\n    # ========================================\n    print(\"\\n\" + \"=\"*40)\n    print(\"RUNNING ENHANCED MODELS\")\n    print(\"=\"*40)\n    \n    train_enhanced, test_enhanced, submission_enhanced, feature_cols = load_data_enhanced()\n    \n    # Run multiple experiments\n    enhanced_results = run_enhanced_experiments(train_enhanced, test_enhanced, submission_enhanced, feature_cols)\n    \n    # Create ensemble\n    ensemble_score = create_ensemble_from_results(enhanced_results, train_enhanced, submission_enhanced)\n    \n    # Analyze overfitting\n    overfitting_analysis = analyze_overfitting(enhanced_results)\n    \n    # ========================================\n    # STEP 3: FINAL SUMMARY\n    # ========================================\n    print(\"\\n\" + \"=\"*60)\n    print(\"FINAL SUMMARY\")\n    print(\"=\"*60)\n    print(f\"Baseline Score:           {baseline_score:.4f}\")\n    print(f\"Enhanced Ensemble Score:  {ensemble_score:.4f}\")\n    print(f\"Improvement:              {ensemble_score - baseline_score:+.4f}\")\n    \n    best_enhanced = max(enhanced_results, key=lambda x: x['oof_score'])\n    print(f\"Best Single Enhanced:     {best_enhanced['oof_score']:.4f} ({best_enhanced['model_name']})\")\n    \n    print(f\"\\nFiles created:\")\n    print(f\"  - submission_baseline.csv\")\n    print(f\"  - submission_enhanced_ensemble.csv\")\n    print(f\"  - overfitting_analysis.csv\")\n    print(f\"  - Individual enhanced model submissions\")\n    \n    return {\n        'baseline_score': baseline_score,\n        'ensemble_score': ensemble_score,\n        'enhanced_results': enhanced_results,\n        'overfitting_analysis': overfitting_analysis\n    }\n\nif __name__ == \"__main__\":\n    results = main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}