{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport json\nimport datetime\nimport math\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple, Optional\nfrom itertools import combinations\nfrom scipy.optimize import minimize\n\n# Machine Learning imports\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler, QuantileTransformer, RobustScaler\nfrom sklearn.feature_selection import mutual_info_regression, SelectKBest\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr, spearmanr\n\n# Deep Learning imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Utilities\nfrom tqdm import tqdm\nimport random\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\n# =========================\n# Directory Setup\n# =========================\nBASE_DIR = Path(\"/kaggle/working\")\nMODEL_DIR = BASE_DIR / \"models\"\nPREDICTIONS_DIR = BASE_DIR / \"predictions\"\nCHECKPOINTS_DIR = BASE_DIR / \"checkpoints\"\nCONFIGS_DIR = BASE_DIR / \"configs\"\nSUBMISSIONS_DIR = BASE_DIR / \"submissions\"\nENSEMBLE_DIR = BASE_DIR / \"ensembles\"\n\n# Create all directories\nfor directory in [MODEL_DIR, PREDICTIONS_DIR, CHECKPOINTS_DIR, CONFIGS_DIR, SUBMISSIONS_DIR, ENSEMBLE_DIR]:\n    directory.mkdir(parents=True, exist_ok=True)\n    print(f\"Created directory: {directory}\")\n\n# =========================\n# Model Configuration\n# =========================\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\", \"bid_qty\",\n        \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\"X817\", \n        \"X586\",  \"X292\"\n    ]\n    \n    MLP_FEATURES = [\n        \"X863\", \"X856\", \"X344\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X137\", \"X855\", \"X174\", \"X302\", \"X178\", \"X532\", \"X168\", \"X612\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n    ]\n    \n    GANDALF_FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"X425\", \"X132\", \"X691\", \"X593\", \"X377\", \"X285\", \"X126\", \"X419\", \"X604\",\n        \"X84\", \"X138\", \"X413\", \"X291\", \"X40\", \"X123\", \"X81\", \"X853\", \"X854\",\n        \"X777\", \"X219\", \"X776\", \"X180\", \"X781\", \"X445\", \"X444\", \"X384\", \"X466\",\n        \"X95\", \"X583\", \"X272\", \"X137\", \"X533\", \"X758\", \"X279\", \"X297\",\n        \"X21\", \"X20\", \"X28\", \"X29\", \"X19\", \"X27\", \"X22\", \"X198\", \"X89\", \"X90\",\n        \"X98\", \"X96\", \"X97\", \"X383\", \"X427\", \"X451\", \"X283\",\n        \"X753\", \"X497\", \"X748\", \"X820\", \"X566\", \"X535\", \"X394\", \"X618\",\n        \"X429\", \"X381\", \"X387\", \"X890\", \"X752\", \"X375\", \"X68\", \"X152\",\n        \"X110\", \"X850\", \"X851\", \"X481\", \"X321\", \"X363\", \"X405\", \"X492\",\n        \"X888\", \"X421\", \"X333\", \"X817\", \"X586\", \"X292\", \"X344\", \"X532\",\n        \"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n    OUTLIER_FRACTION = 0.001\n    \n    # All available models\n    ALL_MODELS = [\"xgboost\", \"mlp\", \"gandalf\", \"simplified_gandalf\", \"anam\", \"dcnv2\", \"tangos\", \"dofen\"]\n\n# XGBoost parameters\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\n# =========================\n# Feature Engineering\n# =========================\ndef add_features(df):\n    \"\"\"Add all engineered features\"\"\"\n    # Original features\n    df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-10)\n    df['log_volume'] = np.log1p(df['volume'])\n\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-10)\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-10)\n    \n    # Price Pressure Indicators\n    df['net_order_flow'] = df['buy_qty'] - df['sell_qty']\n    df['normalized_net_flow'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['buying_pressure'] = df['buy_qty'] / (df['volume'] + 1e-10)\n    df['volume_weighted_buy'] = df['buy_qty'] * df['volume']\n    \n    # Liquidity Depth Measures\n    df['total_depth'] = df['bid_qty'] + df['ask_qty']\n    df['depth_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['relative_spread'] = np.abs(df['bid_qty'] - df['ask_qty']) / (df['total_depth'] + 1e-10)\n    df['log_depth'] = np.log1p(df['total_depth'])\n    \n    # Order Flow Toxicity Proxies\n    df['kyle_lambda'] = np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['flow_toxicity'] = np.abs(df['order_flow_imbalance']) * df['volume']\n    df['aggressive_flow_ratio'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    \n    # Market Activity Indicators\n    df['volume_depth_ratio'] = df['volume'] / (df['total_depth'] + 1e-10)\n    df['activity_intensity'] = (df['buy_qty'] + df['sell_qty']) / (df['volume'] + 1e-10)\n    df['log_buy_qty'] = np.log1p(df['buy_qty'])\n    df['log_sell_qty'] = np.log1p(df['sell_qty'])\n    df['log_bid_qty'] = np.log1p(df['bid_qty'])\n    df['log_ask_qty'] = np.log1p(df['ask_qty'])\n    \n    # Microstructure Volatility Proxies\n    df['realized_spread_proxy'] = 2 * np.abs(df['net_order_flow']) / (df['volume'] + 1e-10)\n    df['price_impact_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10)\n    df['quote_volatility_proxy'] = np.abs(df['depth_imbalance'])\n    \n    # Complex Interaction Terms\n    df['flow_depth_interaction'] = df['net_order_flow'] * df['total_depth']\n    df['imbalance_volume_interaction'] = df['order_flow_imbalance'] * df['volume']\n    df['depth_volume_interaction'] = df['total_depth'] * df['volume']\n    df['buy_sell_spread'] = np.abs(df['buy_qty'] - df['sell_qty'])\n    df['bid_ask_spread'] = np.abs(df['bid_qty'] - df['ask_qty'])\n    \n    # Information Asymmetry Measures\n    df['trade_informativeness'] = df['net_order_flow'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    df['execution_shortfall_proxy'] = df['buy_sell_spread'] / (df['volume'] + 1e-10)\n    df['adverse_selection_proxy'] = df['net_order_flow'] / (df['total_depth'] + 1e-10) * df['volume']\n    \n    # Market Efficiency Indicators\n    df['fill_probability'] = df['volume'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['execution_rate'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_efficiency'] = df['volume'] / (df['bid_ask_spread'] + 1e-10)\n    \n    # Non-linear Transformations\n    df['sqrt_volume'] = np.sqrt(df['volume'])\n    df['sqrt_depth'] = np.sqrt(df['total_depth'])\n    df['volume_squared'] = df['volume'] ** 2\n    df['imbalance_squared'] = df['order_flow_imbalance'] ** 2\n    \n    # Relative Measures\n    df['bid_ratio'] = df['bid_qty'] / (df['total_depth'] + 1e-10)\n    df['ask_ratio'] = df['ask_qty'] / (df['total_depth'] + 1e-10)\n    df['buy_ratio'] = df['buy_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    df['sell_ratio'] = df['sell_qty'] / (df['buy_qty'] + df['sell_qty'] + 1e-10)\n    \n    # Market Stress Indicators\n    df['liquidity_consumption'] = (df['buy_qty'] + df['sell_qty']) / (df['total_depth'] + 1e-10)\n    df['market_stress'] = df['volume'] / (df['total_depth'] + 1e-10) * np.abs(df['order_flow_imbalance'])\n    df['depth_depletion'] = df['volume'] / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n    \n    # Directional Indicators\n    df['net_buying_ratio'] = df['net_order_flow'] / (df['volume'] + 1e-10)\n    df['directional_volume'] = df['net_order_flow'] * np.log1p(df['volume'])\n    df['signed_volume'] = np.sign(df['net_order_flow']) * df['volume']\n    \n    # Replace infinities and NaNs\n    df = df.replace([np.inf, -np.inf], 0).fillna(0)\n    \n    return df\n\n# =========================\n# Utility Functions\n# =========================\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef detect_outliers_and_adjust_weights(X, y, sample_weights, outlier_fraction=0.001):\n    \"\"\"Detect outliers and adjust weights\"\"\"\n    rf = RandomForestRegressor(n_estimators=50, max_depth=10, random_state=42, n_jobs=-1)\n    rf.fit(X, y, sample_weight=sample_weights)\n    \n    predictions = rf.predict(X)\n    residuals = np.abs(y - predictions)\n    \n    n_outliers = max(1, int(len(residuals) * outlier_fraction))\n    threshold = np.sort(residuals)[-n_outliers]\n    \n    outlier_mask = residuals >= threshold\n    adjusted_weights = sample_weights.copy()\n    \n    if outlier_mask.any():\n        outlier_residuals = residuals[outlier_mask]\n        min_outlier_res = outlier_residuals.min()\n        max_outlier_res = outlier_residuals.max()\n        \n        if max_outlier_res > min_outlier_res:\n            normalized_residuals = (outlier_residuals - min_outlier_res) / (max_outlier_res - min_outlier_res)\n        else:\n            normalized_residuals = np.ones_like(outlier_residuals)\n        \n        weight_factors = 0.8 - 0.6 * normalized_residuals\n        adjusted_weights[outlier_mask] *= weight_factors\n        \n        print(f\"    Adjusted weights for {n_outliers} outliers ({outlier_fraction*100:.1f}% of data)\")\n    \n    return adjusted_weights\n\ndef load_data():\n    \"\"\"Load and prepare data\"\"\"\n    all_features = list(set(Config.FEATURES + Config.MLP_FEATURES + Config.GANDALF_FEATURES))\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=all_features + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=all_features)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n\n    train_df = add_features(train_df)\n    test_df = add_features(test_df)\n\n    # Update Config.FEATURES with new features\n    engineered_features = [\n        \"log_volume\", 'bid_ask_interaction', 'bid_buy_interaction', 'bid_sell_interaction', \n        'ask_buy_interaction', 'ask_sell_interaction', 'net_order_flow', 'normalized_net_flow',\n        'buying_pressure', 'volume_weighted_buy', 'total_depth', 'depth_imbalance',\n        'relative_spread', 'log_depth', 'kyle_lambda', 'flow_toxicity', 'aggressive_flow_ratio',\n        'volume_depth_ratio', 'activity_intensity', 'log_buy_qty', 'log_sell_qty',\n        'log_bid_qty', 'log_ask_qty', 'realized_spread_proxy', 'price_impact_proxy',\n        'quote_volatility_proxy', 'flow_depth_interaction', 'imbalance_volume_interaction',\n        'depth_volume_interaction', 'buy_sell_spread', 'bid_ask_spread', 'trade_informativeness',\n        'execution_shortfall_proxy', 'adverse_selection_proxy', 'fill_probability',\n        'execution_rate', 'market_efficiency', 'sqrt_volume', 'sqrt_depth', 'volume_squared',\n        'imbalance_squared', 'bid_ratio', 'ask_ratio', 'buy_ratio', 'sell_ratio',\n        'liquidity_consumption', 'market_stress', 'depth_depletion', 'net_buying_ratio',\n        'directional_volume', 'signed_volume'\n    ]\n    \n    Config.FEATURES += engineered_features\n\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\ndef get_model_slices(n_samples: int):\n    \"\"\"Get model training slices\"\"\"\n    base_slices = [\n        {\"name\": \"full_data\", \"cutoff\": 0, \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_90pct\", \"cutoff\": int(0.10 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_85pct\", \"cutoff\": int(0.15 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"last_80pct\", \"cutoff\": int(0.20 * n_samples), \"is_oldest\": False, \"outlier_adjusted\": False},\n        {\"name\": \"oldest_25pct\", \"cutoff\": int(0.25 * n_samples), \"is_oldest\": True, \"outlier_adjusted\": False},\n    ]\n    \n    outlier_adjusted_slices = []\n    for slice_info in base_slices:\n        adjusted_slice = slice_info.copy()\n        adjusted_slice[\"name\"] = f\"{slice_info['name']}_outlier_adj\"\n        adjusted_slice[\"outlier_adjusted\"] = True\n        outlier_adjusted_slices.append(adjusted_slice)\n    \n    return base_slices + outlier_adjusted_slices\n\ndef save_predictions(model_name: str, predictions: np.ndarray, metadata: dict = None):\n    \"\"\"Save model predictions with metadata\"\"\"\n    pred_path = PREDICTIONS_DIR / f\"{model_name}_predictions.npy\"\n    np.save(pred_path, predictions)\n    \n    # Save metadata - convert numpy types to Python types for JSON serialization\n    meta = {\n        \"model_name\": model_name,\n        \"shape\": list(predictions.shape),\n        \"stats\": {\n            \"mean\": float(np.mean(predictions)),\n            \"std\": float(np.std(predictions)),\n            \"min\": float(np.min(predictions)),\n            \"max\": float(np.max(predictions))\n        },\n        \"saved_at\": datetime.datetime.now().isoformat()\n    }\n    \n    if metadata:\n        # Convert all numpy types to Python types in metadata\n        for key, value in metadata.items():\n            if isinstance(value, np.ndarray):\n                metadata[key] = value.tolist()\n            elif isinstance(value, (np.float32, np.float64)):\n                metadata[key] = float(value)\n            elif isinstance(value, (np.int32, np.int64)):\n                metadata[key] = int(value)\n        meta.update(metadata)\n        \n    meta_path = pred_path.with_suffix('.json')\n    with open(meta_path, 'w') as f:\n        json.dump(meta, f, indent=4)\n        \n    print(f\"💾 Saved {model_name} predictions to {pred_path}\")\n    print(f\"   Mean: {meta['stats']['mean']:.6f}, Std: {meta['stats']['std']:.6f}\")\n\n# =========================\n# Deep Learning Components\n# =========================\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\ndef get_activation_function(name):\n    \"\"\"Return the activation function based on the name.\"\"\"\n    if name == None:\n        return None\n    name = name.lower()\n    if name == 'relu':\n        return nn.ReLU()\n    elif name == 'tanh':\n        return nn.Tanh()\n    elif name == 'sigmoid':\n        return nn.Sigmoid()\n    elif name == 'gelu':\n        return nn.GELU()\n    else:\n        raise ValueError(f\"Unsupported activation function: {name}\")\n\ndef get_dataloaders(X, Y, hparams, device, shuffle=True):\n    \"\"\"Create DataLoader for training and validation datasets.\"\"\"\n    X_tensor = torch.tensor(X, dtype=torch.float32, device=device)\n    if Y is not None:\n        Y_tensor = torch.tensor(Y.values if hasattr(Y, 'values') else Y, \n                                dtype=torch.float32, device=device).unsqueeze(1)\n        dataset = TensorDataset(X_tensor, Y_tensor)\n    else:\n        dataset = TensorDataset(X_tensor)\n    \n    dataloader = DataLoader(dataset, batch_size=hparams[\"batch_size\"], shuffle=shuffle, \n                            generator=torch.Generator().manual_seed(hparams[\"seed\"]))\n    return dataloader\n\nclass Checkpointer:\n    def __init__(self, path=\"best_model.pt\"):\n        self.path = CHECKPOINTS_DIR / path\n        self.best_pearson = -np.inf\n\n    def load(self, model):\n        \"\"\"Load the best model weights.\"\"\"\n        model.load_state_dict(torch.load(self.path))\n        print(f\"Model loaded from {self.path} with best Pearson: {self.best_pearson:.4f}\")\n        return model\n\n    def __call__(self, pearson_coef, model):\n        \"\"\"Save the model if the Pearson coefficient is better than the best one.\"\"\"\n        if pearson_coef > self.best_pearson:\n            self.best_pearson = pearson_coef\n            torch.save(model.state_dict(), self.path)\n            print(f\"✅ New best model saved to {self.path} with Pearson: {pearson_coef:.4f}\")\n\nclass MLP(nn.Module):\n    def __init__(self, dropout_rate=0.6, \n                 layers=[128, 64], activation='relu', last_activation=None):\n        super(MLP, self).__init__()\n        \n        self.linears = nn.ModuleList()\n        self.activation = get_activation_function(activation)\n        self.last_activation = get_activation_function(last_activation)\n\n        for i in range(len(layers) - 1):\n            self.linears.append(nn.Linear(layers[i], layers[i + 1]))\n\n        self.dropout = nn.Dropout(dropout_rate)\n\n    def forward(self, x):\n        for k in range(len(self.linears) - 1):\n            x = self.activation(self.linears[k](x))\n            x = self.dropout(x)\n        x = self.linears[-1](x)\n        if self.last_activation is not None:\n            x = self.last_activation(x)\n        return x\n\n# =========================\n# DCN V2 Components\n# =========================\nclass CrossLayerV2(nn.Module):\n    \"\"\"Cross Layer for DCN V2 with low-rank approximation\"\"\"\n    def __init__(self, input_dim, num_experts=4, low_rank=32):\n        super().__init__()\n        self.input_dim = input_dim\n        self.num_experts = num_experts\n        self.low_rank = low_rank\n        \n        # Expert weights with low-rank decomposition\n        self.U = nn.Parameter(torch.randn(num_experts, input_dim, low_rank))\n        self.V = nn.Parameter(torch.randn(num_experts, low_rank, input_dim))\n        self.C = nn.Parameter(torch.randn(num_experts, low_rank, low_rank))\n        \n        # Gating network\n        self.gate = nn.Sequential(\n            nn.Linear(input_dim, 128),\n            nn.ReLU(),\n            nn.Linear(128, num_experts),\n            nn.Softmax(dim=-1)\n        )\n        \n        # Bias\n        self.bias = nn.Parameter(torch.zeros(input_dim))\n        \n        # Initialize\n        nn.init.xavier_uniform_(self.U)\n        nn.init.xavier_uniform_(self.V)\n        nn.init.xavier_uniform_(self.C)\n        \n    def forward(self, x0, x):\n        \"\"\"\n        x0: input to the cross layer (batch_size, input_dim)\n        x: current layer input (batch_size, input_dim)\n        \"\"\"\n        batch_size = x.shape[0]\n        \n        # Compute gating weights\n        gates = self.gate(x)  # (batch_size, num_experts)\n        \n        # Compute expert outputs\n        expert_outputs = []\n        for i in range(self.num_experts):\n            # W = U * C * V\n            W_i = torch.matmul(torch.matmul(self.U[i], self.C[i]), self.V[i])\n            # x0 * (W * x + b)\n            output_i = x0 * (torch.matmul(x, W_i.t()) + self.bias)\n            expert_outputs.append(output_i)\n        \n        # Stack expert outputs\n        expert_outputs = torch.stack(expert_outputs, dim=1)  # (batch_size, num_experts, input_dim)\n        \n        # Apply gating\n        gates = gates.unsqueeze(-1)  # (batch_size, num_experts, 1)\n        output = torch.sum(expert_outputs * gates, dim=1)  # (batch_size, input_dim)\n        \n        return output + x  # Residual connection\n\nclass DCNV2(nn.Module):\n    \"\"\"Deep & Cross Network V2\"\"\"\n    def __init__(self, input_dim, num_cross_layers=3, num_experts=4, \n                 deep_dims=[256, 128, 64], dropout=0.2):\n        super().__init__()\n        \n        self.input_dim = input_dim\n        self.num_cross_layers = num_cross_layers\n        \n        # Cross network\n        self.cross_layers = nn.ModuleList()\n        for _ in range(num_cross_layers):\n            self.cross_layers.append(\n                CrossLayerV2(input_dim, num_experts=num_experts, low_rank=32)\n            )\n        \n        # Deep network\n        deep_layers = []\n        prev_dim = input_dim\n        \n        for dim in deep_dims:\n            deep_layers.extend([\n                nn.Linear(prev_dim, dim),\n                nn.BatchNorm1d(dim),\n                nn.ReLU(),\n                nn.Dropout(dropout)\n            ])\n            prev_dim = dim\n            \n        self.deep_network = nn.Sequential(*deep_layers)\n        self.deep_output_dim = prev_dim\n        \n        # Combination layer\n        self.combination = nn.Sequential(\n            nn.Linear(input_dim + self.deep_output_dim, 128),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(128, 1)\n        )\n        \n    def forward(self, x):\n        # Cross network\n        x0 = x\n        cross_output = x\n        for cross_layer in self.cross_layers:\n            cross_output = cross_layer(x0, cross_output)\n        \n        # Deep network\n        deep_output = self.deep_network(x)\n        \n        # Combine cross and deep\n        combined = torch.cat([cross_output, deep_output], dim=1)\n        output = self.combination(combined)\n        \n        return output\n\nclass DifferentiableDecisionTree(nn.Module):\n    \"\"\"Soft decision tree for GANDALF with improved numerical stability\"\"\"\n    def __init__(self, input_dim, depth, temperature=1.0):\n        super().__init__()\n        self.depth = depth\n        self.n_leaves = 2 ** depth\n        \n        # Internal nodes with better initialization\n        self.internal_nodes = nn.ModuleList()\n        for i in range(2 ** depth - 1):\n            linear = nn.Linear(input_dim, 1)\n            # Xavier initialization scaled down\n            nn.init.xavier_uniform_(linear.weight, gain=0.5)\n            nn.init.zeros_(linear.bias)\n            self.internal_nodes.append(linear)\n        \n        # Leaf values with small initialization\n        self.leaf_values = nn.Parameter(torch.zeros(self.n_leaves))\n        nn.init.normal_(self.leaf_values, mean=0, std=0.01)\n        \n        # Temperature as learnable parameter with constraints\n        self.log_temperature = nn.Parameter(torch.log(torch.tensor(temperature)))\n        \n    def forward(self, x):\n        batch_size = x.size(0)\n        device = x.device\n        \n        # Constrain temperature to reasonable range\n        temp = torch.clamp(torch.exp(self.log_temperature), min=0.1, max=10.0)\n        \n        # Initialize path probabilities\n        path_probs = torch.ones(batch_size, 1, device=device)\n        \n        # Traverse the tree\n        for level in range(self.depth):\n            n_nodes = 2 ** level\n            next_path_probs = []\n            \n            for node in range(n_nodes):\n                node_idx = 2 ** level - 1 + node\n                \n                if node_idx < len(self.internal_nodes) and node < path_probs.size(1):\n                    # Get decision logit\n                    logit = self.internal_nodes[node_idx](x).squeeze(-1)\n                    \n                    # Stable sigmoid with temperature\n                    logit_scaled = logit / temp\n                    # Clamp to prevent overflow\n                    logit_scaled = torch.clamp(logit_scaled, min=-10, max=10)\n                    split_prob = torch.sigmoid(logit_scaled)\n                    \n                    current_prob = path_probs[:, node:node+1]\n                    \n                    # Left and right probabilities\n                    left_prob = current_prob * (1 - split_prob).unsqueeze(1)\n                    right_prob = current_prob * split_prob.unsqueeze(1)\n                    \n                    next_path_probs.append(left_prob)\n                    next_path_probs.append(right_prob)\n            \n            if next_path_probs:\n                path_probs = torch.cat(next_path_probs, dim=1)\n        \n        # Ensure we have the right number of leaf probabilities\n        if path_probs.size(1) != self.n_leaves:\n            # Pad or truncate as needed\n            if path_probs.size(1) < self.n_leaves:\n                padding = torch.zeros(batch_size, self.n_leaves - path_probs.size(1), device=device)\n                path_probs = torch.cat([path_probs, padding], dim=1)\n            else:\n                path_probs = path_probs[:, :self.n_leaves]\n        \n        # Normalize path probabilities for numerical stability\n        path_probs = path_probs + 1e-10\n        path_probs = path_probs / path_probs.sum(dim=1, keepdim=True)\n        \n        # Compute output as weighted sum of leaf values\n        output = torch.sum(path_probs * self.leaf_values.unsqueeze(0), dim=1)\n        \n        return output\n\nclass GatingNetwork(nn.Module):\n    \"\"\"Gating network for tree selection with improved stability\"\"\"\n    def __init__(self, input_dim, n_trees, hidden_dim=128, dropout=0.3):\n        super().__init__()\n        \n        self.network = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.BatchNorm1d(hidden_dim // 2),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim // 2, n_trees)\n        )\n        \n        # Initialize last layer with smaller weights\n        nn.init.xavier_uniform_(self.network[-1].weight, gain=0.1)\n        \n    def forward(self, x):\n        gates = self.network(x)\n        # Temperature softmax for more stable gradients\n        gates = F.softmax(gates / 2.0, dim=-1)\n        return gates\n\nclass GANDALF(nn.Module):\n    \"\"\"GANDALF: Gated Additive Neural Decision Additive Forest with improved stability\"\"\"\n    def __init__(self, config):\n        super().__init__()\n        \n        self.n_trees = config['n_trees']\n        self.tree_depth = config['tree_depth']\n        self.input_dim = config['input_dim']\n        \n        # Feature embedding with batch normalization\n        embed_layers = []\n        prev_dim = self.input_dim\n        \n        # First layer with batch norm\n        embed_layers.extend([\n            nn.Linear(prev_dim, config['embed_dims'][0]),\n            nn.BatchNorm1d(config['embed_dims'][0]),\n            nn.GELU(),\n            nn.Dropout(config['feature_dropout'])\n        ])\n        prev_dim = config['embed_dims'][0]\n        \n        # Remaining layers\n        for dim in config['embed_dims'][1:]:\n            embed_layers.extend([\n                nn.Linear(prev_dim, dim),\n                nn.BatchNorm1d(dim),\n                nn.GELU(),\n                nn.Dropout(config['feature_dropout'])\n            ])\n            prev_dim = dim\n            \n        self.feature_embedder = nn.Sequential(*embed_layers)\n        self.embed_dim = prev_dim\n        \n        # Decision trees with varying depths\n        self.trees = nn.ModuleList()\n        for i in range(self.n_trees):\n            # Vary tree depth slightly\n            tree_depth = self.tree_depth + (i % 3 - 1)\n            tree_depth = max(2, min(tree_depth, 6))  # Constrain depth\n            \n            self.trees.append(\n                DifferentiableDecisionTree(\n                    self.embed_dim,\n                    tree_depth,\n                    temperature=config['tree_temperature']\n                )\n            )\n        \n        # Gating network\n        self.gating_network = GatingNetwork(\n            self.input_dim,\n            self.n_trees,\n            config['gate_hidden_dim'],\n            config['gate_dropout']\n        )\n        \n        # Optional neural head\n        if config.get('use_nn_head', True):\n            head_layers = []\n            prev_dim = self.input_dim\n            \n            for dim in config['head_dims']:\n                head_layers.extend([\n                    nn.Linear(prev_dim, dim),\n                    nn.BatchNorm1d(dim),\n                    nn.GELU(),\n                    nn.Dropout(config['head_dropout'])\n                ])\n                prev_dim = dim\n                \n            head_layers.append(nn.Linear(prev_dim, 1))\n            self.nn_head = nn.Sequential(*head_layers)\n            \n            # Combination weight initialized to favor trees\n            self.combination_weight = nn.Parameter(torch.tensor(0.7))\n        else:\n            self.nn_head = None\n            \n        # Output scaling parameters\n        self.output_scale = nn.Parameter(torch.ones(1))\n        self.output_bias = nn.Parameter(torch.zeros(1))\n            \n    def forward(self, x):\n        # Embed features\n        embedded = self.feature_embedder(x)\n        \n        # Get tree outputs\n        tree_outputs = []\n        for tree in self.trees:\n            output = tree(embedded)\n            tree_outputs.append(output)\n        \n        # Stack tree outputs\n        tree_outputs = torch.stack(tree_outputs, dim=1)  # (batch_size, n_trees)\n        \n        # Get gating weights\n        gates = self.gating_network(x)  # (batch_size, n_trees)\n        \n        # Weighted sum of tree outputs\n        forest_output = torch.sum(gates * tree_outputs, dim=1, keepdim=True)\n        \n        # Combine with neural head if available\n        if self.nn_head is not None:\n            nn_output = self.nn_head(x)\n            weight = torch.sigmoid(self.combination_weight)\n            final_output = weight * forest_output + (1 - weight) * nn_output\n        else:\n            final_output = forest_output\n            \n        # Scale and shift output\n        final_output = final_output * self.output_scale + self.output_bias\n            \n        return final_output\n\nclass SimplifiedGANDALF(nn.Module):\n    \"\"\"Simplified GANDALF model with better numerical stability\"\"\"\n    def __init__(self, config):\n        super().__init__()\n        \n        self.input_dim = config['input_dim']\n        self.n_estimators = config.get('n_estimators', 20)\n        self.tree_dim = config.get('tree_dim', 128)\n        self.depth = config.get('depth', 3)\n        \n        # Feature transformation layers\n        self.feature_layers = nn.Sequential(\n            nn.Linear(self.input_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, self.tree_dim),\n            nn.BatchNorm1d(self.tree_dim),\n            nn.ReLU(),\n            nn.Dropout(0.2)\n        )\n        \n        # Soft decision trees as simple neural networks\n        self.trees = nn.ModuleList()\n        for _ in range(self.n_estimators):\n            tree = nn.Sequential(\n                nn.Linear(self.tree_dim, 64),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(64, 32),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(32, 1)\n            )\n            self.trees.append(tree)\n        \n        # Gating network\n        self.gate = nn.Sequential(\n            nn.Linear(self.input_dim, 128),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(128, self.n_estimators),\n            nn.Softmax(dim=1)\n        )\n        \n        # Output combination\n        self.output_weight = nn.Parameter(torch.ones(1))\n        self.output_bias = nn.Parameter(torch.zeros(1))\n        \n    def forward(self, x):\n        # Transform features\n        tree_features = self.feature_layers(x)\n        \n        # Get tree outputs\n        tree_outputs = []\n        for tree in self.trees:\n            output = tree(tree_features)\n            tree_outputs.append(output)\n        \n        tree_outputs = torch.cat(tree_outputs, dim=1)  # (batch_size, n_estimators)\n        \n        # Get gates\n        gates = self.gate(x)  # (batch_size, n_estimators)\n        \n        # Weighted combination\n        output = torch.sum(tree_outputs * gates, dim=1, keepdim=True)\n        \n        # Scale and shift\n        output = output * self.output_weight + self.output_bias\n        \n        return output\n\nclass ANAM(nn.Module):\n    \"\"\"Additive Neural Attention Model\"\"\"\n    def __init__(self, n_features, shape_hidden_dim=64, shape_n_hidden=2, \n                 attention_hidden_dim=128, n_heads=4, dropout=0.2):\n        super().__init__()\n        \n        self.n_features = n_features\n        self.n_heads = n_heads\n        \n        # Shape functions for each feature\n        self.shape_functions = nn.ModuleList()\n        for _ in range(n_features):\n            layers = []\n            in_dim = 1\n            \n            for i in range(shape_n_hidden):\n                layers.extend([\n                    nn.Linear(in_dim, shape_hidden_dim),\n                    nn.LayerNorm(shape_hidden_dim),\n                    nn.ReLU(),\n                    nn.Dropout(0.1)\n                ])\n                in_dim = shape_hidden_dim\n            \n            layers.append(nn.Linear(shape_hidden_dim, 1))\n            self.shape_functions.append(nn.Sequential(*layers))\n        \n        # Feature embeddings for attention\n        self.feature_embeddings = nn.Parameter(torch.randn(n_features, attention_hidden_dim))\n        nn.init.xavier_uniform_(self.feature_embeddings)\n        \n        # Multi-head attention\n        self.attention = nn.MultiheadAttention(\n            embed_dim=attention_hidden_dim,\n            num_heads=n_heads,\n            dropout=dropout,\n            batch_first=True\n        )\n        \n        # Context vector for attention\n        self.context = nn.Parameter(torch.randn(1, attention_hidden_dim))\n        nn.init.xavier_uniform_(self.context)\n        \n        # Final aggregation\n        self.output_layer = nn.Sequential(\n            nn.Linear(n_features, attention_hidden_dim),\n            nn.LayerNorm(attention_hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(attention_hidden_dim, 1)\n        )\n        \n        # Learnable bias\n        self.bias = nn.Parameter(torch.zeros(1))\n        \n    def forward(self, x, return_components=False):\n        batch_size = x.size(0)\n        \n        # Apply shape functions to each feature\n        shape_outputs = []\n        for i in range(self.n_features):\n            feature_input = x[:, i:i+1]  # (batch_size, 1)\n            shape_output = self.shape_functions[i](feature_input)\n            shape_outputs.append(shape_output)\n        \n        shape_outputs = torch.cat(shape_outputs, dim=1)  # (batch_size, n_features)\n        \n        # Prepare for attention\n        # Expand feature embeddings for batch\n        feature_embeds = self.feature_embeddings.unsqueeze(0).expand(batch_size, -1, -1)\n        \n        # Expand context for batch\n        context = self.context.expand(batch_size, -1, -1)\n        \n        # Apply attention\n        attended_features, attention_weights = self.attention(\n            query=context,\n            key=feature_embeds,\n            value=feature_embeds\n        )\n        \n        # Attention weights shape: (batch_size, 1, n_features)\n        attention_weights = attention_weights.squeeze(1)  # (batch_size, n_features)\n        \n        # Apply attention weights to shape outputs\n        weighted_outputs = shape_outputs * attention_weights\n        \n        # Final output\n        output = self.output_layer(weighted_outputs) + self.bias\n        \n        if return_components:\n            return output, shape_outputs, attention_weights\n        \n        return output\n    \n    def get_feature_importance(self, x):\n        \"\"\"Calculate feature importance based on attention weights\"\"\"\n        self.eval()\n        with torch.no_grad():\n            _, _, attention_weights = self.forward(x, return_components=True)\n            # Average attention weights across batch\n            importance = attention_weights.mean(dim=0)\n        return importance\n\n# =========================\n# TANGOS Model Components\n# =========================\nclass GatedFeatureExtractor(nn.Module):\n    \"\"\"Gated mechanism for feature extraction with smoothing\"\"\"\n    def __init__(self, input_dim, hidden_dim, dropout=0.2):\n        super().__init__()\n        \n        # Gate networks\n        self.gate_transform = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.LayerNorm(hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, hidden_dim),\n            nn.Sigmoid()\n        )\n        \n        # Feature transformation\n        self.feature_transform = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.LayerNorm(hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, hidden_dim)\n        )\n        \n        # Smoothing layer\n        self.smoothing = nn.Sequential(\n            nn.Linear(hidden_dim, hidden_dim),\n            nn.LayerNorm(hidden_dim),\n            nn.Tanh()\n        )\n        \n    def forward(self, x):\n        # Apply gating\n        gates = self.gate_transform(x)\n        features = self.feature_transform(x)\n        \n        # Gated features\n        gated_features = gates * features\n        \n        # Apply smoothing\n        smoothed = self.smoothing(gated_features)\n        \n        # Residual connection\n        return smoothed + gated_features\n\nclass TemporalInspiredBlock(nn.Module):\n    \"\"\"Temporal-inspired processing block even though this is regression\"\"\"\n    def __init__(self, input_dim, hidden_dim, n_layers=2, dropout=0.2):\n        super().__init__()\n        \n        self.layers = nn.ModuleList()\n        self.norms = nn.ModuleList()\n        \n        for i in range(n_layers):\n            layer_input_dim = input_dim if i == 0 else hidden_dim\n            \n            # GRU-inspired gating without sequential dependency\n            self.layers.append(nn.Linear(layer_input_dim, hidden_dim * 3))\n            self.norms.append(nn.LayerNorm(hidden_dim))\n            \n        self.dropout = nn.Dropout(dropout)\n        self.output_projection = nn.Linear(hidden_dim, hidden_dim)\n        \n    def forward(self, x):\n        h = x\n        \n        for layer, norm in zip(self.layers, self.norms):\n            # GRU-style gating\n            gates = layer(h)\n            i, f, g = gates.chunk(3, dim=-1)\n            \n            i = torch.sigmoid(i)  # Input gate\n            f = torch.sigmoid(f)  # Forget gate  \n            g = torch.tanh(g)     # Candidate\n            \n            # Update hidden state\n            if h.shape[-1] == i.shape[-1]:\n                h = f * h + i * g\n            else:\n                h = i * g\n                \n            h = norm(h)\n            h = self.dropout(h)\n        \n        return self.output_projection(h)\n\nclass SmoothingAttention(nn.Module):\n    \"\"\"Attention mechanism with built-in smoothing\"\"\"\n    def __init__(self, hidden_dim, n_heads=8, dropout=0.2, temperature=1.0):\n        super().__init__()\n        \n        self.hidden_dim = hidden_dim\n        self.n_heads = n_heads\n        self.head_dim = hidden_dim // n_heads\n        self.temperature = temperature\n        \n        self.q_proj = nn.Linear(hidden_dim, hidden_dim)\n        self.k_proj = nn.Linear(hidden_dim, hidden_dim)\n        self.v_proj = nn.Linear(hidden_dim, hidden_dim)\n        \n        self.dropout = nn.Dropout(dropout)\n        self.out_proj = nn.Linear(hidden_dim, hidden_dim)\n        \n        # Smoothing parameters\n        self.smooth_alpha = nn.Parameter(torch.ones(1) * 0.1)\n        \n    def forward(self, x, mask=None):\n        batch_size, seq_len, _ = x.shape\n        \n        # Linear projections\n        Q = self.q_proj(x).view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2)\n        K = self.k_proj(x).view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2)\n        V = self.v_proj(x).view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2)\n        \n        # Scaled dot-product attention with temperature\n        scores = torch.matmul(Q, K.transpose(-2, -1)) / (math.sqrt(self.head_dim) * self.temperature)\n        \n        if mask is not None:\n            scores = scores.masked_fill(mask == 0, -1e9)\n        \n        # Apply smoothing to attention scores\n        attn_weights = F.softmax(scores, dim=-1)\n        \n        # Smooth the attention weights\n        smoothing_kernel = torch.ones(1, 1, 3, device=x.device) / 3\n        if attn_weights.dim() == 4 and seq_len > 2:\n            # Apply 1D smoothing across sequence dimension\n            attn_weights_smooth = F.conv1d(\n                attn_weights.view(-1, 1, seq_len),\n                smoothing_kernel,\n                padding=1\n            ).view(batch_size, self.n_heads, seq_len, seq_len)\n            \n            # Blend original and smoothed weights\n            attn_weights = (1 - torch.sigmoid(self.smooth_alpha)) * attn_weights + \\\n                          torch.sigmoid(self.smooth_alpha) * attn_weights_smooth\n        \n        attn_weights = self.dropout(attn_weights)\n        \n        # Apply attention\n        context = torch.matmul(attn_weights, V)\n        context = context.transpose(1, 2).contiguous().view(batch_size, seq_len, self.hidden_dim)\n        \n        output = self.out_proj(context)\n        \n        return output, attn_weights\n\nclass TANGOS(nn.Module):\n    \"\"\"Temporal Attention Networks with Gated Operations and Smoothing\"\"\"\n    def __init__(self, config):\n        super().__init__()\n        \n        self.input_dim = config['input_dim']\n        self.hidden_dim = config['hidden_dim']\n        self.n_layers = config.get('n_layers', 3)\n        self.n_heads = config.get('n_heads', 8)\n        self.dropout = config.get('dropout', 0.2)\n        self.use_feature_gating = config.get('use_feature_gating', True)\n        \n        # Input projection with batch norm\n        self.input_projection = nn.Sequential(\n            nn.Linear(self.input_dim, self.hidden_dim),\n            nn.BatchNorm1d(self.hidden_dim),\n            nn.GELU(),\n            nn.Dropout(self.dropout)\n        )\n        \n        # Gated feature extractor\n        if self.use_feature_gating:\n            self.feature_gating = GatedFeatureExtractor(\n                self.hidden_dim, \n                self.hidden_dim, \n                dropout=self.dropout\n            )\n        \n        # Temporal-inspired blocks\n        self.temporal_blocks = nn.ModuleList()\n        for _ in range(self.n_layers):\n            self.temporal_blocks.append(\n                TemporalInspiredBlock(\n                    self.hidden_dim,\n                    self.hidden_dim,\n                    n_layers=2,\n                    dropout=self.dropout\n                )\n            )\n        \n        # Smoothing attention layers\n        self.attention_layers = nn.ModuleList()\n        for _ in range(self.n_layers):\n            self.attention_layers.append(\n                SmoothingAttention(\n                    self.hidden_dim,\n                    n_heads=self.n_heads,\n                    dropout=self.dropout,\n                    temperature=1.5\n                )\n            )\n        \n        # Layer normalization after each block\n        self.layer_norms = nn.ModuleList([\n            nn.LayerNorm(self.hidden_dim) for _ in range(self.n_layers)\n        ])\n        \n        # Global pooling strategies\n        self.pooling_weight = nn.Parameter(torch.ones(3) / 3)\n        \n        # Output layers with smoothing\n        self.output_smooth = nn.Sequential(\n            nn.Linear(self.hidden_dim * 3, self.hidden_dim),\n            nn.LayerNorm(self.hidden_dim),\n            nn.GELU(),\n            nn.Dropout(self.dropout),\n            nn.Linear(self.hidden_dim, 128),\n            nn.LayerNorm(128),\n            nn.GELU(),\n            nn.Dropout(self.dropout),\n            nn.Linear(128, 1)\n        )\n        \n        # Learnable smoothing for final output\n        self.output_smooth_param = nn.Parameter(torch.tensor(0.1))\n        self.running_mean = None\n        self.alpha = 0.95  # EMA alpha\n        \n    def forward(self, x):\n        batch_size = x.size(0)\n        \n        # Input projection\n        h = self.input_projection(x)\n        \n        # Apply feature gating if enabled\n        if self.use_feature_gating:\n            h = self.feature_gating(h)\n        \n        # Add sequence dimension for attention\n        h = h.unsqueeze(1)  # (batch_size, 1, hidden_dim)\n        \n        # Process through temporal and attention blocks\n        for i in range(self.n_layers):\n            # Temporal processing\n            h_temp = self.temporal_blocks[i](h.squeeze(1)).unsqueeze(1)\n            \n            # Attention with residual\n            h_attn, _ = self.attention_layers[i](h)\n            \n            # Combine temporal and attention outputs\n            h = h + h_temp + h_attn\n            h = self.layer_norms[i](h)\n        \n        # Multi-scale pooling\n        h_squeezed = h.squeeze(1)\n        \n        # Different pooling strategies\n        max_pool = h_squeezed\n        avg_pool = h_squeezed\n        last_hidden = h_squeezed\n        \n        # Weighted combination of pooling strategies\n        weights = F.softmax(self.pooling_weight, dim=0)\n        pooled = torch.cat([\n            max_pool * weights[0],\n            avg_pool * weights[1],\n            last_hidden * weights[2]\n        ], dim=-1)\n        \n        # Final output\n        output = self.output_smooth(pooled)\n        \n        # Apply output smoothing during training\n        if self.training and self.running_mean is not None:\n            # Exponential moving average smoothing\n            smoothed_output = (1 - self.alpha) * output + self.alpha * self.running_mean\n            self.running_mean = output.detach().mean()\n            \n            # Blend raw and smoothed outputs\n            smooth_weight = torch.sigmoid(self.output_smooth_param)\n            output = (1 - smooth_weight) * output + smooth_weight * smoothed_output\n        elif self.training:\n            # Initialize running mean\n            self.running_mean = output.detach().mean()\n        \n        return output\n\n# =========================\n# DOFEN Model Components\n# =========================\nclass ObliviousDecisionTree(nn.Module):\n    \"\"\"Differentiable Oblivious Decision Tree\"\"\"\n    def __init__(self, depth, num_features, temperature=1.0):\n        super().__init__()\n        self.depth = depth\n        self.num_features = num_features\n        self.temperature = temperature\n        self.num_leaves = 2 ** depth\n        \n        # Internal node parameters\n        self.feature_indices = nn.Parameter(torch.randn(depth, num_features))\n        self.thresholds = nn.Parameter(torch.randn(depth))\n        \n        # Leaf values\n        self.leaf_values = nn.Parameter(torch.randn(self.num_leaves))\n        \n        # Initialize parameters\n        nn.init.xavier_uniform_(self.feature_indices)\n        nn.init.zeros_(self.thresholds)\n        nn.init.normal_(self.leaf_values, mean=0, std=0.01)\n        \n    def forward(self, x):\n        batch_size = x.shape[0]\n        \n        # Soft routing through the tree\n        decisions = []\n        for d in range(self.depth):\n            # Soft feature selection using softmax\n            feature_weights = F.softmax(self.feature_indices[d] / self.temperature, dim=0)\n            selected_feature = (x * feature_weights.unsqueeze(0)).sum(dim=1)\n            \n            # Soft threshold comparison\n            decision = torch.sigmoid((selected_feature - self.thresholds[d]) / self.temperature)\n            decisions.append(decision)\n        \n        # Compute leaf probabilities (soft routing)\n        leaf_probs = torch.ones(batch_size, self.num_leaves, device=x.device)\n        for d in range(self.depth):\n            decision = decisions[d].unsqueeze(1)\n            # Update probabilities for left and right subtrees\n            mask = torch.zeros(batch_size, self.num_leaves, device=x.device)\n            for leaf in range(self.num_leaves):\n                if (leaf >> (self.depth - 1 - d)) & 1:\n                    mask[:, leaf] = 1\n            leaf_probs = leaf_probs * (mask * decision + (1 - mask) * (1 - decision))\n        \n        # Normalize probabilities for stability\n        leaf_probs = leaf_probs + 1e-10\n        leaf_probs = leaf_probs / leaf_probs.sum(dim=1, keepdim=True)\n        \n        # Weighted sum of leaf values\n        output = (leaf_probs * self.leaf_values.unsqueeze(0)).sum(dim=1)\n        \n        return output\n\nclass SparseFeatureSelection(nn.Module):\n    \"\"\"Sparse column selection layer for DOFEN\"\"\"\n    def __init__(self, num_features, num_selected, temperature=1.0):\n        super().__init__()\n        self.num_features = num_features\n        self.num_selected = num_selected\n        self.temperature = temperature\n        \n        # Feature selection scores\n        self.scores = nn.Parameter(torch.randn(num_features))\n        nn.init.normal_(self.scores, mean=0, std=0.1)\n        \n    def forward(self, x):\n        # Compute selection probabilities\n        if self.training:\n            # Add Gumbel noise for exploration during training\n            gumbel_noise = -torch.log(-torch.log(torch.rand_like(self.scores) + 1e-10) + 1e-10)\n            scores_with_noise = (self.scores + gumbel_noise) / self.temperature\n        else:\n            scores_with_noise = self.scores / self.temperature\n        \n        # Get top-k features (soft selection)\n        if self.num_selected < self.num_features:\n            _, indices = torch.topk(scores_with_noise, self.num_selected)\n            mask = torch.zeros_like(self.scores)\n            mask.scatter_(0, indices, 1.0)\n            \n            # Soft mask for gradient flow\n            soft_mask = torch.sigmoid(self.scores * 5)\n            if not self.training:\n                mask = soft_mask\n        else:\n            mask = torch.ones_like(self.scores)\n        \n        # Apply mask with small residual connection for stability\n        selected_features = x * mask.unsqueeze(0) + x * 0.1\n        \n        return selected_features, mask\n\nclass ObliviousForestLayer(nn.Module):\n    \"\"\"Single layer of oblivious forest\"\"\"\n    def __init__(self, num_trees, depth, num_features, temperature=1.0):\n        super().__init__()\n        self.num_trees = num_trees\n        self.num_features = num_features\n        self.trees = nn.ModuleList([\n            ObliviousDecisionTree(depth, num_features, temperature)\n            for _ in range(num_trees)\n        ])\n        \n        # Tree weights for ensemble\n        self.tree_weights = nn.Parameter(torch.ones(num_trees) / num_trees)\n        \n    def forward(self, x):\n        # Ensemble predictions\n        tree_outputs = []\n        for tree in self.trees:\n            tree_outputs.append(tree(x).unsqueeze(1))\n        \n        tree_outputs = torch.cat(tree_outputs, dim=1)  # (batch_size, num_trees)\n        \n        # Weighted ensemble\n        weights = F.softmax(self.tree_weights, dim=0)\n        output = (tree_outputs * weights.unsqueeze(0)).sum(dim=1)\n        \n        return output, tree_outputs\n\nclass DOFEN(nn.Module):\n    \"\"\"Deep Oblivious Forest ENsemble\"\"\"\n    def __init__(self, num_features, num_layers=2, trees_per_layer=[50, 25], \n                 tree_depth=4, feature_subset_ratio=0.6, temperature=1.5,\n                 use_sparse_selection=True, dropout=0.2):\n        super().__init__()\n        self.num_features = num_features\n        self.num_layers = num_layers\n        self.use_sparse_selection = use_sparse_selection\n        \n        # Input normalization\n        self.input_norm = nn.BatchNorm1d(num_features)\n        \n        # Sparse feature selection\n        if use_sparse_selection:\n            num_selected = int(num_features * feature_subset_ratio)\n            self.feature_selector = SparseFeatureSelection(num_features, num_selected, temperature)\n        \n        # Forest layers\n        self.forest_layers = nn.ModuleList()\n        for i in range(num_layers):\n            layer = ObliviousForestLayer(\n                num_trees=trees_per_layer[i],\n                depth=tree_depth,\n                num_features=num_features,\n                temperature=temperature\n            )\n            self.forest_layers.append(layer)\n        \n        # Layer normalization\n        self.layer_norms = nn.ModuleList([\n            nn.LayerNorm(1) for _ in range(num_layers)\n        ])\n        \n        # Combination weights\n        self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)\n        \n        # Final projection\n        self.final_layer = nn.Sequential(\n            nn.Linear(num_layers, 128),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(128, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(64, 1)\n        )\n        \n        # Output scaling\n        self.output_scale = nn.Parameter(torch.ones(1))\n        self.output_bias = nn.Parameter(torch.zeros(1))\n        \n    def forward(self, x):\n        # Input normalization\n        x = self.input_norm(x)\n        \n        # Feature selection\n        if self.use_sparse_selection:\n            x_selected, feature_mask = self.feature_selector(x)\n        else:\n            x_selected = x\n        \n        # Forward through forest layers\n        layer_outputs = []\n        \n        for i, (layer, norm) in enumerate(zip(self.forest_layers, self.layer_norms)):\n            layer_output, _ = layer(x_selected)\n            layer_output = norm(layer_output.unsqueeze(1)).squeeze(1)\n            layer_outputs.append(layer_output.unsqueeze(1))\n        \n        # Combine layer outputs\n        layer_outputs = torch.cat(layer_outputs, dim=1)  # (batch_size, num_layers)\n        \n        # Weighted combination\n        weights = F.softmax(self.layer_weights, dim=0)\n        weighted_output = (layer_outputs * weights.unsqueeze(0)).sum(dim=1, keepdim=True)\n        \n        # Final prediction through neural layers\n        output = self.final_layer(layer_outputs).squeeze()\n        \n        # Combine neural output with weighted forest output\n        final_output = 0.7 * output + 0.3 * weighted_output.squeeze()\n        \n        # Apply scaling\n        final_output = final_output * self.output_scale + self.output_bias\n        \n        return final_output\n\n# =========================\n# XGBoost Training\n# =========================\ndef train_xgboost(train_df, test_df):\n    \"\"\"Train XGBoost with multiple slices\"\"\"\n    print(\"\\n=== Training XGBoost Model ===\")\n    \n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    oof_preds = {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n    test_preds = {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            is_oldest = s[\"is_oldest\"]\n            outlier_adjusted = s[\"outlier_adjusted\"]\n            \n            if is_oldest:\n                subset = train_df.iloc[:cutoff].reset_index(drop=True)\n                rel_idx = train_idx[train_idx < cutoff]\n                sw = np.ones(len(rel_idx))\n            else:\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            \n            if outlier_adjusted and len(X_train) > 100:\n                sw = detect_outliers_and_adjust_weights(\n                    X_train.values, \n                    y_train.values, \n                    sw, \n                    outlier_fraction=Config.OUTLIER_FRACTION\n                )\n\n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n\n            model = XGBRegressor(**XGB_PARAMS)\n            model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n            if is_oldest:\n                oof_preds[slice_name][valid_idx] = model.predict(\n                    train_df.iloc[valid_idx][Config.FEATURES]\n                )\n            else:\n                mask = valid_idx >= cutoff\n                if mask.any():\n                    idxs = valid_idx[mask]\n                    oof_preds[slice_name][idxs] = model.predict(\n                        train_df.iloc[idxs][Config.FEATURES]\n                    )\n                if cutoff > 0 and (~mask).any():\n                    base_slice_name = slice_name.replace(\"_outlier_adj\", \"\")\n                    if base_slice_name == slice_name:\n                        fallback_slice = \"full_data\"\n                    else:\n                        fallback_slice = \"full_data_outlier_adj\"\n                    oof_preds[slice_name][valid_idx[~mask]] = oof_preds[fallback_slice][valid_idx[~mask]]\n\n            test_preds[slice_name] += model.predict(test_df[Config.FEATURES])\n\n    # Normalize test predictions\n    for slice_name in test_preds:\n        test_preds[slice_name] /= (Config.N_FOLDS - 1)\n\n    # Create XGBoost ensemble\n    weights = np.array([\n        1.0,   # full_data\n        1.0,   # last_90pct\n        1.0,   # last_85pct\n        1.0,   # last_80pct\n        0.25,  # oldest_25pct\n        0.9,   # full_data_outlier_adj\n        0.9,   # last_90pct_outlier_adj\n        0.9,   # last_85pct_outlier_adj\n        0.9,   # last_80pct_outlier_adj\n        0.2    # oldest_25pct_outlier_adj\n    ])\n    \n    weights = weights / weights.sum()\n\n    oof_weighted = pd.DataFrame(oof_preds).values @ weights\n    test_weighted = pd.DataFrame(test_preds).values @ weights\n    score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], oof_weighted)[0]\n    print(f\"\\nXGBoost Weighted Ensemble Pearson: {score_weighted:.4f}\")\n\n    # Print individual slice scores\n    print(\"\\nIndividual slice OOF scores and weights:\")\n    slice_names = list(oof_preds.keys())\n    slice_scores = {}\n    for i, slice_name in enumerate(slice_names):\n        score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[slice_name])[0]\n        slice_scores[slice_name] = score\n        print(f\"  {slice_name}: {score:.4f} (weight: {weights[i]:.3f})\")\n\n    # Save predictions and metadata\n    save_predictions(\"xgboost\", test_weighted, {\n        \"slice_scores\": slice_scores,\n        \"ensemble_weights\": weights.tolist(),\n        \"oof_score\": score_weighted\n    })\n    \n    # Also save OOF predictions for ensemble analysis\n    np.save(PREDICTIONS_DIR / \"xgboost_oof.npy\", oof_weighted)\n    \n    return test_weighted, oof_weighted\n\n# =========================\n# MLP Training\n# =========================\ndef train_mlp(train_df, test_df):\n    \"\"\"Train MLP model\"\"\"\n    print(\"\\n=== Training MLP Model ===\")\n    \n    hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 10,\n        \"batch_size\": 1024 * 8 * 4,\n        \"learning_rate\": 0.001,\n        \"weight_decay\": 1e-3,\n        \"dropout_rate\": 0.6,\n        \"layers\": [len(Config.MLP_FEATURES), 256, 64, 1],\n        \"hidden_activation\": None,\n        \"activation\": \"relu\",\n        \"delta\": 5,\n        \"noise_factor\": 0.005\n    }\n    \n    set_seed(hparams[\"seed\"])\n    \n    X_train_full = train_df[Config.MLP_FEATURES].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    # Keep track of OOF predictions\n    oof_preds = np.zeros(len(train_df))\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    # Collect all fold predictions for ensemble\n    test_preds_all_folds = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_full), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        \n        X_train = X_train_full[train_idx]\n        y_train = y_train_full[train_idx]\n        X_val = X_train_full[valid_idx]\n        y_val = y_train_full[valid_idx]\n        \n        scaler = StandardScaler()\n        X_train = scaler.fit_transform(X_train)\n        X_val = scaler.transform(X_val)\n        X_test = scaler.transform(test_df[Config.MLP_FEATURES].values)\n        \n        # Create dataloaders\n        train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n        val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n        test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n        \n        model = MLP(\n            layers=hparams[\"layers\"],\n            dropout_rate=hparams[\"dropout_rate\"],\n            activation=hparams[\"activation\"],\n            last_activation=hparams[\"hidden_activation\"],\n        ).to(device)\n        \n        criterion = nn.HuberLoss(delta=hparams[\"delta\"], reduction='sum')\n        optimizer = optim.Adam(model.parameters(), lr=hparams[\"learning_rate\"], \n                              weight_decay=hparams[\"weight_decay\"])\n        \n        checkpoint_path = CHECKPOINTS_DIR / f\"mlp_fold{fold}_model.pt\"\n        best_pearson = -np.inf\n        \n        # Training loop\n        num_epochs = hparams[\"num_epochs\"]\n        \n        for epoch in range(num_epochs):\n            model.train()\n            running_loss = 0.0\n\n            for inputs, targets in train_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                inputs = inputs + torch.randn_like(inputs) * hparams[\"noise_factor\"]\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                loss.backward()\n                optimizer.step()\n                \n                running_loss += loss.item() * inputs.size(0)\n                \n            running_loss = running_loss / len(train_loader.dataset)\n\n            # Validation\n            model.eval()\n            val_loss = 0.0\n            preds = []\n            trues = []\n            with torch.no_grad():\n                for inputs, targets in val_loader:\n                    inputs, targets = inputs.to(device), targets.to(device)\n                    outputs = model(inputs)\n                    loss = criterion(outputs, targets)\n                    val_loss += loss.item() * inputs.size(0)\n                    preds.append(outputs.cpu().numpy())\n                    trues.append(targets.cpu().numpy())\n\n            val_loss /= len(val_loader.dataset)\n            preds = np.concatenate(preds).flatten()\n            trues = np.concatenate(trues).flatten()\n            pearson_coef = pearsonr(preds, trues)[0]\n\n            if pearson_coef > best_pearson:\n                best_pearson = pearson_coef\n                torch.save(model.state_dict(), checkpoint_path)\n                \n        # Load best model and save OOF predictions\n        model.load_state_dict(torch.load(checkpoint_path))\n        model.eval()\n        \n        val_preds = []\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs = inputs.to(device)\n                outputs = model(inputs)\n                val_preds.append(outputs.cpu().numpy())\n        \n        oof_preds[valid_idx] = np.concatenate(val_preds).flatten()\n        \n        # Make test predictions\n        test_preds = []\n        with torch.no_grad():\n            for inputs in test_loader:\n                inputs = inputs[0].to(device)\n                outputs = model(inputs)\n                test_preds.append(outputs.cpu().numpy())\n        \n        test_preds_all_folds.append(np.concatenate(test_preds).flatten())\n    \n    # Average test predictions across folds\n    test_predictions = np.mean(test_preds_all_folds, axis=0)\n    \n    # Calculate overall OOF score\n    oof_score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds)[0]\n    print(f\"\\nMLP Overall OOF Pearson: {oof_score:.4f}\")\n    \n    # Save predictions\n    save_predictions(\"mlp\", test_predictions, {\n        \"oof_score\": oof_score,\n        \"n_folds\": Config.N_FOLDS\n    })\n    \n    # Save OOF predictions for ensemble analysis\n    np.save(PREDICTIONS_DIR / \"mlp_oof.npy\", oof_preds)\n    \n    return test_predictions, oof_preds\n\n# =========================\n# DCN V2 Training\n# =========================\ndef train_dcnv2(train_df, test_df):\n    \"\"\"Train DCN V2 model\"\"\"\n    print(\"\\n=== Training DCN V2 Model ===\")\n    \n    hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 15,\n        \"batch_size\": 512,\n        \"learning_rate\": 0.001,\n        \"weight_decay\": 1e-4,\n        \"num_cross_layers\": 3,\n        \"num_experts\": 4,\n        \"deep_dims\": [256, 128, 64],\n        \"dropout\": 0.2,\n        \"gradient_clip\": 1.0,\n        \"patience\": 5,\n        \"noise_factor\": 0.005\n    }\n    \n    set_seed(hparams[\"seed\"])\n    \n    # Use MLP features for DCN V2\n    X_train_full = train_df[Config.MLP_FEATURES].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    print(f\"Using {len(Config.MLP_FEATURES)} features for DCN V2\")\n    \n    # Keep track of OOF predictions\n    oof_preds = np.zeros(len(train_df))\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    # Collect all fold predictions\n    test_preds_all_folds = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_full), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        \n        X_train = X_train_full[train_idx]\n        y_train = y_train_full[train_idx]\n        X_val = X_train_full[valid_idx]\n        y_val = y_train_full[valid_idx]\n        \n        # Scale data\n        scaler = StandardScaler()\n        X_train = scaler.fit_transform(X_train)\n        X_val = scaler.transform(X_val)\n        X_test = scaler.transform(test_df[Config.MLP_FEATURES].values)\n        \n        # Create dataloaders\n        train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n        val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n        test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n        \n        # Initialize model\n        model = DCNV2(\n            input_dim=len(Config.MLP_FEATURES),\n            num_cross_layers=hparams[\"num_cross_layers\"],\n            num_experts=hparams[\"num_experts\"],\n            deep_dims=hparams[\"deep_dims\"],\n            dropout=hparams[\"dropout\"]\n        ).to(device)\n        \n        criterion = nn.HuberLoss(delta=1.0, reduction='mean')\n        optimizer = optim.AdamW(model.parameters(), lr=hparams[\"learning_rate\"], \n                                weight_decay=hparams[\"weight_decay\"])\n        \n        scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2, min_lr=1e-6)\n        \n        checkpoint_path = CHECKPOINTS_DIR / f\"dcnv2_fold{fold}_model.pt\"\n        best_pearson = -np.inf\n        patience_counter = 0\n        \n        # Training loop\n        num_epochs = hparams[\"num_epochs\"]\n        for epoch in range(num_epochs):\n            model.train()\n            running_loss = 0.0\n            \n            for inputs, targets in train_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                \n                # Add noise for robustness\n                inputs = inputs + torch.randn_like(inputs) * hparams[\"noise_factor\"]\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                loss.backward()\n                \n                # Gradient clipping\n                torch.nn.utils.clip_grad_norm_(model.parameters(), hparams[\"gradient_clip\"])\n                \n                optimizer.step()\n                \n                running_loss += loss.item() * inputs.size(0)\n            \n            running_loss = running_loss / len(train_loader.dataset)\n\n            # Validation phase\n            model.eval()\n            val_loss = 0.0\n            preds = []\n            trues = []\n            with torch.no_grad():\n                for inputs, targets in val_loader:\n                    inputs, targets = inputs.to(device), targets.to(device)\n                    outputs = model(inputs)\n                    loss = criterion(outputs, targets)\n                    val_loss += loss.item() * inputs.size(0)\n                    preds.append(outputs.cpu().numpy())\n                    trues.append(targets.cpu().numpy())\n\n            val_loss /= len(val_loader.dataset)\n            preds = np.concatenate(preds).flatten()\n            trues = np.concatenate(trues).flatten()\n            pearson_coef = pearsonr(preds, trues)[0]\n\n            scheduler.step(pearson_coef)\n            \n            # Save best model\n            if pearson_coef > best_pearson:\n                best_pearson = pearson_coef\n                torch.save(model.state_dict(), checkpoint_path)\n                patience_counter = 0\n            else:\n                patience_counter += 1\n                if patience_counter >= hparams[\"patience\"]:\n                    break\n        \n        # Load best model and save OOF predictions\n        model.load_state_dict(torch.load(checkpoint_path))\n        model.eval()\n        \n        val_preds = []\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs = inputs.to(device)\n                outputs = model(inputs)\n                val_preds.append(outputs.cpu().numpy())\n        \n        oof_preds[valid_idx] = np.concatenate(val_preds).flatten()\n        \n        # Make test predictions\n        predictions = []\n        with torch.no_grad():\n            for inputs in test_loader:\n                inputs = inputs[0].to(device)\n                outputs = model(inputs)\n                predictions.append(outputs.cpu().numpy())\n\n        test_preds_all_folds.append(np.concatenate(predictions).flatten())\n    \n    # Average test predictions across folds\n    test_predictions = np.mean(test_preds_all_folds, axis=0)\n    \n    # Calculate overall OOF score\n    oof_score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds)[0]\n    print(f\"\\nDCN V2 Overall OOF Pearson: {oof_score:.4f}\")\n    \n    # Save predictions\n    save_predictions(\"dcnv2\", test_predictions, {\n        \"oof_score\": oof_score,\n        \"n_features\": len(Config.MLP_FEATURES)\n    })\n    \n    # Save OOF predictions for ensemble analysis\n    np.save(PREDICTIONS_DIR / \"dcnv2_oof.npy\", oof_preds)\n    \n    return test_predictions, oof_preds\n\n# =========================\n# GANDALF Training\n# =========================\ndef train_gandalf_model(model, train_loader, val_loader, config, device, checkpoint_path):\n    \"\"\"Train GANDALF model with improved stability\"\"\"\n    \n    criterion = nn.HuberLoss(delta=config.get('huber_delta', 1.0))\n    optimizer = optim.AdamW(\n        model.parameters(),\n        lr=config['learning_rate'],\n        weight_decay=config['weight_decay'],\n        betas=(0.9, 0.999),\n        eps=1e-8\n    )\n    \n    scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=3, min_lr=1e-6)\n    \n    best_val_pearson = -np.inf\n    patience_counter = 0\n    patience = config.get('patience', 10)\n    num_epochs = config.get('num_epochs', 30)\n    \n    saved_checkpoint = False\n    \n    # Use mixed precision training if available\n    use_amp = config.get('use_amp', True) and device.type == 'cuda'\n    scaler = GradScaler() if use_amp else None\n    \n    for epoch in range(num_epochs):\n        model.train()\n        train_loss = 0.0\n        train_batches = 0\n        \n        progress_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs}\")\n        for inputs, targets in progress_bar:\n            inputs, targets = inputs.to(device), targets.to(device)\n            \n            # Add noise for regularization\n            if config.get('noise_factor', 0) > 0:\n                noise = torch.randn_like(inputs) * config['noise_factor']\n                inputs = inputs + noise\n            \n            optimizer.zero_grad()\n            \n            if use_amp:\n                with autocast():\n                    outputs = model(inputs)\n                    loss = criterion(outputs, targets)\n                \n                # Check for NaN\n                if torch.isnan(loss):\n                    print(f\"NaN loss detected at epoch {epoch+1}, skipping batch\")\n                    continue\n                    \n                scaler.scale(loss).backward()\n                scaler.unscale_(optimizer)\n                torch.nn.utils.clip_grad_norm_(model.parameters(), config.get('grad_clip', 1.0))\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                # Check for NaN\n                if torch.isnan(loss):\n                    print(f\"NaN loss detected at epoch {epoch+1}, skipping batch\")\n                    continue\n                \n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(model.parameters(), config.get('grad_clip', 1.0))\n                optimizer.step()\n            \n            train_loss += loss.item()\n            train_batches += 1\n            progress_bar.set_postfix({'loss': f'{loss.item():.4f}'})\n        \n        if train_batches == 0:\n            print(f\"No valid batches in epoch {epoch+1}, stopping training\")\n            break\n        \n        # Validation\n        model.eval()\n        val_loss = 0.0\n        val_preds = []\n        val_targets = []\n        val_batches = 0\n        \n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                if not torch.isnan(loss):\n                    val_loss += loss.item()\n                    val_preds.extend(outputs.cpu().numpy().flatten())\n                    val_targets.extend(targets.cpu().numpy().flatten())\n                    val_batches += 1\n        \n        if val_batches == 0:\n            print(f\"No valid validation batches, stopping training\")\n            break\n        \n        avg_train_loss = train_loss / train_batches\n        avg_val_loss = val_loss / val_batches\n        \n        if len(val_preds) > 0:\n            val_pearson = pearsonr(val_targets, val_preds)[0]\n            val_spearman = spearmanr(val_targets, val_preds)[0]\n        else:\n            val_pearson = -np.inf\n            val_spearman = -np.inf\n        \n        print(f\"\\nTrain Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}\")\n        print(f\"Val Pearson: {val_pearson:.4f}, Val Spearman: {val_spearman:.4f}\")\n        \n        scheduler.step(val_pearson)\n        \n        if not np.isnan(val_pearson) and val_pearson > best_val_pearson:\n            best_val_pearson = val_pearson\n            patience_counter = 0\n            torch.save(model.state_dict(), checkpoint_path)\n            saved_checkpoint = True\n            print(f\"✅ New best model saved! Pearson: {best_val_pearson:.4f}\")\n        else:\n            patience_counter += 1\n            if patience_counter >= patience:\n                print(f\"Early stopping triggered after {epoch+1} epochs\")\n                break\n    \n    # Load best model if saved\n    if saved_checkpoint and checkpoint_path.exists():\n        model.load_state_dict(torch.load(checkpoint_path))\n    \n    return model, best_val_pearson\n\ndef train_gandalf(train_df, test_df):\n    \"\"\"Train both GANDALF and Simplified GANDALF models\"\"\"\n    print(\"\\n=== Training GANDALF Models ===\")\n    \n    set_seed(42)\n    \n    # Prepare features\n    gandalf_features = Config.GANDALF_FEATURES.copy()\n    engineered_features = [\n        \"log_volume\", 'bid_ask_interaction', 'net_order_flow', 'normalized_net_flow',\n        'buying_pressure', 'total_depth', 'depth_imbalance', 'kyle_lambda', \n        'aggressive_flow_ratio', 'volume_depth_ratio', 'log_buy_qty', 'log_sell_qty',\n        'price_impact_proxy', 'market_stress', 'liquidity_consumption'\n    ]\n    \n    all_gandalf_features = gandalf_features + engineered_features\n    all_gandalf_features = list(set(all_gandalf_features))\n    all_gandalf_features = [f for f in all_gandalf_features if f in train_df.columns]\n    \n    print(f\"Using {len(all_gandalf_features)} features for GANDALF\")\n    \n    # Feature selection\n    print(\"\\nSelecting features...\")\n    selector = SelectKBest(score_func=mutual_info_regression, k=min(80, len(all_gandalf_features)))\n    X_train_full = train_df[all_gandalf_features].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    X_train_selected = selector.fit_transform(X_train_full, y_train_full)\n    selected_features = [all_gandalf_features[i] for i in selector.get_support(indices=True)]\n    print(f\"Selected {len(selected_features)} features\")\n    \n    # Keep track of OOF predictions for both models\n    oof_preds_gandalf = np.zeros(len(train_df))\n    oof_preds_simplified = np.zeros(len(train_df))\n    \n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    # Collect all fold predictions\n    test_preds_gandalf_folds = []\n    test_preds_simplified_folds = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_selected), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        \n        X_train = X_train_selected[train_idx]\n        y_train = y_train_full[train_idx]\n        X_val = X_train_selected[valid_idx]\n        y_val = y_train_full[valid_idx]\n        \n        # Transform data using RobustScaler for better stability\n        transformer = RobustScaler()\n        X_train_transformed = transformer.fit_transform(X_train)\n        X_val_transformed = transformer.transform(X_val)\n        \n        # Clip extreme values\n        clip_value = 5.0\n        X_train_transformed = np.clip(X_train_transformed, -clip_value, clip_value)\n        X_val_transformed = np.clip(X_val_transformed, -clip_value, clip_value)\n        \n        # Create data loaders\n        train_dataset = TensorDataset(\n            torch.tensor(X_train_transformed, dtype=torch.float32),\n            torch.tensor(y_train, dtype=torch.float32).unsqueeze(1)\n        )\n        val_dataset = TensorDataset(\n            torch.tensor(X_val_transformed, dtype=torch.float32),\n            torch.tensor(y_val, dtype=torch.float32).unsqueeze(1)\n        )\n        \n        train_loader = DataLoader(train_dataset, batch_size=256, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=1024, shuffle=False)\n        \n        # 1. Train Original GANDALF\n        print(f\"\\n=== Training Original GANDALF - Fold {fold} ===\")\n        \n        gandalf_config = {\n            'input_dim': X_train_transformed.shape[1],\n            'n_trees': 10,\n            'tree_depth': 4,\n            'tree_temperature': 1.5,\n            'embed_dims': [128, 64],\n            'feature_dropout': 0.2,\n            'gate_hidden_dim': 64,\n            'gate_dropout': 0.2,\n            'use_nn_head': True,\n            'head_dims': [128, 64],\n            'head_dropout': 0.3,\n            'learning_rate': 0.0001,\n            'weight_decay': 0.01,\n            'huber_delta': 1.0,\n            'noise_factor': 0.005,\n            'grad_clip': 0.5,\n            'num_epochs': 15,\n            'patience': 5,\n            'use_amp': True\n        }\n        \n        gandalf_model = GANDALF(gandalf_config).to(device)\n        gandalf_checkpoint = CHECKPOINTS_DIR / f\"gandalf_fold{fold}_model.pt\"\n        gandalf_model, gandalf_score = train_gandalf_model(\n            gandalf_model, train_loader, val_loader, gandalf_config, device, gandalf_checkpoint\n        )\n        \n        # 2. Train Simplified GANDALF\n        print(f\"\\n=== Training Simplified GANDALF - Fold {fold} ===\")\n        \n        simplified_config = {\n            'input_dim': X_train_transformed.shape[1],\n            'n_estimators': 15,\n            'tree_dim': 64,\n            'depth': 3,\n            'learning_rate': 0.0005,\n            'weight_decay': 0.01,\n            'huber_delta': 1.0,\n            'noise_factor': 0.01,\n            'grad_clip': 0.5,\n            'num_epochs': 15,\n            'patience': 5\n        }\n        \n        simplified_model = SimplifiedGANDALF(simplified_config).to(device)\n        simplified_checkpoint = CHECKPOINTS_DIR / f\"simplified_gandalf_fold{fold}_model.pt\"\n        simplified_model, simplified_score = train_gandalf_model(\n            simplified_model, train_loader, val_loader, simplified_config, device, simplified_checkpoint\n        )\n        \n        # Make validation predictions\n        gandalf_model.eval()\n        simplified_model.eval()\n        \n        val_preds_gandalf = []\n        val_preds_simplified = []\n        \n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs = inputs.to(device)\n                \n                outputs_gandalf = gandalf_model(inputs)\n                val_preds_gandalf.extend(outputs_gandalf.cpu().numpy().flatten())\n                \n                outputs_simplified = simplified_model(inputs)\n                val_preds_simplified.extend(outputs_simplified.cpu().numpy().flatten())\n        \n        oof_preds_gandalf[valid_idx] = np.array(val_preds_gandalf)\n        oof_preds_simplified[valid_idx] = np.array(val_preds_simplified)\n        \n        # Make test predictions\n        X_test = test_df[all_gandalf_features].values\n        X_test_selected = selector.transform(X_test)\n        X_test_transformed = transformer.transform(X_test_selected)\n        X_test_transformed = np.clip(X_test_transformed, -clip_value, clip_value)\n        \n        test_dataset = TensorDataset(torch.tensor(X_test_transformed, dtype=torch.float32))\n        test_loader = DataLoader(test_dataset, batch_size=2048, shuffle=False)\n        \n        gandalf_predictions = []\n        simplified_predictions = []\n        \n        with torch.no_grad():\n            for (inputs,) in test_loader:\n                inputs = inputs.to(device)\n                \n                outputs_gandalf = gandalf_model(inputs)\n                gandalf_predictions.extend(outputs_gandalf.cpu().numpy().flatten())\n                \n                outputs_simplified = simplified_model(inputs)\n                simplified_predictions.extend(outputs_simplified.cpu().numpy().flatten())\n        \n        test_preds_gandalf_folds.append(np.array(gandalf_predictions))\n        test_preds_simplified_folds.append(np.array(simplified_predictions))\n    \n    # Average test predictions across folds\n    test_gandalf_final = np.mean(test_preds_gandalf_folds, axis=0)\n    test_simplified_final = np.mean(test_preds_simplified_folds, axis=0)\n    \n    # Post-processing\n    pred_mean = train_df[Config.LABEL_COLUMN].mean()\n    pred_std = train_df[Config.LABEL_COLUMN].std()\n    \n    test_gandalf_final = np.clip(test_gandalf_final, pred_mean - 4 * pred_std, pred_mean + 4 * pred_std)\n    test_simplified_final = np.clip(test_simplified_final, pred_mean - 4 * pred_std, pred_mean + 4 * pred_std)\n    \n    # Calculate OOF scores\n    oof_score_gandalf = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds_gandalf)[0]\n    oof_score_simplified = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds_simplified)[0]\n    \n    print(f\"\\nGANDALF Overall OOF Pearson: {oof_score_gandalf:.4f}\")\n    print(f\"Simplified GANDALF Overall OOF Pearson: {oof_score_simplified:.4f}\")\n    \n    # Save predictions\n    save_predictions(\"gandalf\", test_gandalf_final, {\n        \"oof_score\": oof_score_gandalf,\n        \"n_features\": len(selected_features),\n        \"model_type\": \"original_differentiable_trees\"\n    })\n    \n    save_predictions(\"simplified_gandalf\", test_simplified_final, {\n        \"oof_score\": oof_score_simplified,\n        \"n_features\": len(selected_features),\n        \"model_type\": \"simplified_neural_trees\"\n    })\n    \n    # Save OOF predictions\n    np.save(PREDICTIONS_DIR / \"gandalf_oof.npy\", oof_preds_gandalf)\n    np.save(PREDICTIONS_DIR / \"simplified_gandalf_oof.npy\", oof_preds_simplified)\n    \n    return test_gandalf_final, test_simplified_final, oof_preds_gandalf, oof_preds_simplified\n\n# =========================\n# ANAM Training\n# =========================\ndef train_anam(train_df, test_df):\n    \"\"\"Train ANAM model\"\"\"\n    print(\"\\n=== Training ANAM Model ===\")\n    \n    # Hyperparameters\n    hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 20,\n        \"batch_size\": 256,\n        \"learning_rate\": 0.001,\n        \"weight_decay\": 1e-4,\n        \"shape_hidden_dim\": 64,\n        \"shape_n_hidden\": 2,\n        \"attention_hidden_dim\": 128,\n        \"n_heads\": 4,\n        \"dropout\": 0.2,\n        \"gradient_clip\": 1.0,\n        \"warmup_epochs\": 5,\n        \"patience\": 5,\n        \"noise_factor\": 0.005\n    }\n    \n    set_seed(hparams[\"seed\"])\n    \n    # Prepare data for ANAM using MLP features\n    X_train_full = train_df[Config.MLP_FEATURES].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    print(f\"Using {len(Config.MLP_FEATURES)} features for ANAM\")\n    \n    # Keep track of OOF predictions\n    oof_preds = np.zeros(len(train_df))\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    # Collect all fold predictions\n    test_preds_all_folds = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_full), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        \n        X_train = X_train_full[train_idx]\n        y_train = y_train_full[train_idx]\n        X_val = X_train_full[valid_idx]\n        y_val = y_train_full[valid_idx]\n        \n        # Scale data\n        scaler = StandardScaler()\n        X_train = scaler.fit_transform(X_train)\n        X_val = scaler.transform(X_val)\n        X_test = scaler.transform(test_df[Config.MLP_FEATURES].values)\n        \n        # Create dataloaders\n        train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n        val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n        test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n        \n        # Initialize model\n        model = ANAM(\n            n_features=len(Config.MLP_FEATURES),\n            shape_hidden_dim=hparams[\"shape_hidden_dim\"],\n            shape_n_hidden=hparams[\"shape_n_hidden\"],\n            attention_hidden_dim=hparams[\"attention_hidden_dim\"],\n            n_heads=hparams[\"n_heads\"],\n            dropout=hparams[\"dropout\"]\n        ).to(device)\n        \n        criterion = nn.HuberLoss(delta=1.0, reduction='mean')\n        optimizer = optim.AdamW(model.parameters(), lr=hparams[\"learning_rate\"], \n                                weight_decay=hparams[\"weight_decay\"])\n        \n        # Learning rate scheduler with warmup\n        def lr_lambda(epoch):\n            if epoch < hparams[\"warmup_epochs\"]:\n                return (epoch + 1) / hparams[\"warmup_epochs\"]\n            else:\n                return 0.5 * (1 + math.cos(math.pi * (epoch - hparams[\"warmup_epochs\"]) / \n                                          (hparams[\"num_epochs\"] - hparams[\"warmup_epochs\"])))\n        \n        scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n        \n        checkpoint_path = CHECKPOINTS_DIR / f\"anam_fold{fold}_model.pt\"\n        best_val_pearson = -np.inf\n        patience_counter = 0\n        \n        # Training loop\n        num_epochs = hparams[\"num_epochs\"]\n        for epoch in range(num_epochs):\n            model.train()\n            running_loss = 0.0\n            \n            for inputs, targets in train_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                \n                # Add noise for robustness\n                inputs = inputs + torch.randn_like(inputs) * hparams[\"noise_factor\"]\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                loss.backward()\n                \n                # Gradient clipping\n                torch.nn.utils.clip_grad_norm_(model.parameters(), hparams[\"gradient_clip\"])\n                \n                optimizer.step()\n                \n                running_loss += loss.item() * inputs.size(0)\n            \n            scheduler.step()\n            \n            running_loss = running_loss / len(train_loader.dataset)\n\n            # Validation phase\n            model.eval()\n            val_loss = 0.0\n            preds = []\n            trues = []\n            with torch.no_grad():\n                for inputs, targets in val_loader:\n                    inputs, targets = inputs.to(device), targets.to(device)\n                    outputs = model(inputs)\n                    loss = criterion(outputs, targets)\n                    val_loss += loss.item() * inputs.size(0)\n                    preds.append(outputs.cpu().numpy())\n                    trues.append(targets.cpu().numpy())\n\n            val_loss /= len(val_loader.dataset)\n            preds = np.concatenate(preds).flatten()\n            trues = np.concatenate(trues).flatten()\n            pearson_coef = pearsonr(preds, trues)[0]\n\n            if pearson_coef > best_val_pearson:\n                best_val_pearson = pearson_coef\n                torch.save(model.state_dict(), checkpoint_path)\n                patience_counter = 0\n            else:\n                patience_counter += 1\n                if patience_counter >= hparams[\"patience\"]:\n                    break\n        \n        # Load best model and save OOF predictions\n        model.load_state_dict(torch.load(checkpoint_path))\n        model.eval()\n        \n        val_preds = []\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs = inputs.to(device)\n                outputs = model(inputs)\n                val_preds.append(outputs.cpu().numpy())\n        \n        oof_preds[valid_idx] = np.concatenate(val_preds).flatten()\n        \n        # Make test predictions\n        predictions = []\n        with torch.no_grad():\n            for inputs in test_loader:\n                inputs = inputs[0].to(device)\n                outputs = model(inputs)\n                predictions.append(outputs.cpu().numpy())\n\n        test_preds_all_folds.append(np.concatenate(predictions).flatten())\n    \n    # Average test predictions across folds\n    test_predictions = np.mean(test_preds_all_folds, axis=0)\n    \n    # Calculate overall OOF score\n    oof_score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds)[0]\n    print(f\"\\nANAM Overall OOF Pearson: {oof_score:.4f}\")\n    \n    # Save predictions\n    save_predictions(\"anam\", test_predictions, {\n        \"oof_score\": oof_score,\n        \"n_features\": len(Config.MLP_FEATURES)\n    })\n    \n    # Save OOF predictions\n    np.save(PREDICTIONS_DIR / \"anam_oof.npy\", oof_preds)\n    \n    return test_predictions, oof_preds\n\n# =========================\n# TANGOS Training\n# =========================\ndef train_tangos(train_df, test_df):\n    \"\"\"Train TANGOS model\"\"\"\n    print(\"\\n=== Training TANGOS Model ===\")\n    \n    # Hyperparameters\n    hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 15,\n        \"batch_size\": 512,\n        \"learning_rate\": 0.0005,\n        \"weight_decay\": 1e-4,\n        \"hidden_dim\": 256,\n        \"n_layers\": 3,\n        \"n_heads\": 8,\n        \"dropout\": 0.25,\n        \"gradient_clip\": 0.5,\n        \"patience\": 5,\n        \"noise_factor\": 0.01,\n        \"use_feature_gating\": True,\n        \"warmup_steps\": 1000\n    }\n    \n    set_seed(hparams[\"seed\"])\n    \n    # Prepare features - use GANDALF features for TANGOS\n    tangos_features = Config.GANDALF_FEATURES.copy()\n    \n    # Add select engineered features\n    engineered_features = [\n        \"log_volume\", \"net_order_flow\", \"normalized_net_flow\",\n        \"total_depth\", \"depth_imbalance\", \"kyle_lambda\",\n        \"volume_depth_ratio\", \"price_impact_proxy\",\n        \"market_stress\", \"liquidity_consumption\"\n    ]\n    \n    all_tangos_features = tangos_features + engineered_features\n    all_tangos_features = list(set(all_tangos_features))\n    all_tangos_features = [f for f in all_tangos_features if f in train_df.columns]\n    \n    print(f\"Using {len(all_tangos_features)} features for TANGOS\")\n    \n    # Use recent data with time decay\n    X_train_full = train_df[all_tangos_features].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    # Keep track of OOF predictions\n    oof_preds = np.zeros(len(train_df))\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    # Collect all fold predictions\n    test_preds_all_folds = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_full), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        \n        X_train = X_train_full[train_idx]\n        y_train = y_train_full[train_idx]\n        X_val = X_train_full[valid_idx]\n        y_val = y_train_full[valid_idx]\n        \n        # Use RobustScaler for outlier resistance\n        scaler = RobustScaler()\n        X_train = scaler.fit_transform(X_train)\n        X_val = scaler.transform(X_val)\n        X_test = scaler.transform(test_df[all_tangos_features].values)\n        \n        # Clip extreme values\n        clip_value = 5.0\n        X_train = np.clip(X_train, -clip_value, clip_value)\n        X_val = np.clip(X_val, -clip_value, clip_value)\n        X_test = np.clip(X_test, -clip_value, clip_value)\n        \n        # Create dataloaders\n        train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n        val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n        test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n        \n        # Initialize model\n        config = {\n            'input_dim': len(all_tangos_features),\n            'hidden_dim': hparams['hidden_dim'],\n            'n_layers': hparams['n_layers'],\n            'n_heads': hparams['n_heads'],\n            'dropout': hparams['dropout'],\n            'use_feature_gating': hparams['use_feature_gating']\n        }\n        \n        model = TANGOS(config).to(device)\n        \n        # Loss and optimizer\n        criterion = nn.HuberLoss(delta=1.0, reduction='mean')\n        optimizer = optim.AdamW(\n            model.parameters(), \n            lr=hparams[\"learning_rate\"],\n            weight_decay=hparams[\"weight_decay\"],\n            betas=(0.9, 0.999)\n        )\n        \n        # Warmup scheduler\n        def lr_lambda(step):\n            if step < hparams[\"warmup_steps\"]:\n                return step / hparams[\"warmup_steps\"]\n            return 1.0\n        \n        scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n        \n        checkpoint_path = CHECKPOINTS_DIR / f\"tangos_fold{fold}_model.pt\"\n        best_pearson = -np.inf\n        patience_counter = 0\n        \n        # Training loop\n        global_step = 0\n        num_epochs = hparams[\"num_epochs\"]\n        \n        for epoch in range(num_epochs):\n            model.train()\n            running_loss = 0.0\n            \n            for inputs, targets in train_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                \n                # Add noise for robustness\n                if hparams[\"noise_factor\"] > 0:\n                    inputs = inputs + torch.randn_like(inputs) * hparams[\"noise_factor\"]\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                loss.backward()\n                \n                # Gradient clipping\n                torch.nn.utils.clip_grad_norm_(model.parameters(), hparams[\"gradient_clip\"])\n                \n                optimizer.step()\n                scheduler.step()\n                \n                running_loss += loss.item() * inputs.size(0)\n                global_step += 1\n            \n            running_loss = running_loss / len(train_loader.dataset)\n\n            # Validation\n            model.eval()\n            val_loss = 0.0\n            preds = []\n            trues = []\n            \n            with torch.no_grad():\n                for inputs, targets in val_loader:\n                    inputs, targets = inputs.to(device), targets.to(device)\n                    outputs = model(inputs)\n                    loss = criterion(outputs, targets)\n                    val_loss += loss.item() * inputs.size(0)\n                    preds.append(outputs.cpu().numpy())\n                    trues.append(targets.cpu().numpy())\n\n            val_loss /= len(val_loader.dataset)\n            preds = np.concatenate(preds).flatten()\n            trues = np.concatenate(trues).flatten()\n            pearson_coef = pearsonr(preds, trues)[0]\n\n            # Save best model\n            if pearson_coef > best_pearson:\n                best_pearson = pearson_coef\n                torch.save(model.state_dict(), checkpoint_path)\n                patience_counter = 0\n            else:\n                patience_counter += 1\n                if patience_counter >= hparams[\"patience\"]:\n                    break\n        \n        # Load best model and save OOF predictions\n        model.load_state_dict(torch.load(checkpoint_path))\n        model.eval()\n        \n        val_preds = []\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs = inputs.to(device)\n                outputs = model(inputs)\n                val_preds.append(outputs.cpu().numpy())\n        \n        oof_preds[valid_idx] = np.concatenate(val_preds).flatten()\n        \n        # Make test predictions\n        predictions = []\n        with torch.no_grad():\n            for inputs in test_loader:\n                inputs = inputs[0].to(device)\n                outputs = model(inputs)\n                predictions.append(outputs.cpu().numpy())\n\n        test_preds_all_folds.append(np.concatenate(predictions).flatten())\n    \n    # Average test predictions across folds\n    test_predictions = np.mean(test_preds_all_folds, axis=0)\n    \n    # Post-process predictions\n    pred_mean = train_df[Config.LABEL_COLUMN].mean()\n    pred_std = train_df[Config.LABEL_COLUMN].std()\n    test_predictions = np.clip(test_predictions, pred_mean - 4 * pred_std, pred_mean + 4 * pred_std)\n    \n    # Calculate overall OOF score\n    oof_score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds)[0]\n    print(f\"\\nTANGOS Overall OOF Pearson: {oof_score:.4f}\")\n    \n    # Save predictions\n    save_predictions(\"tangos\", test_predictions, {\n        \"oof_score\": oof_score,\n        \"n_features\": len(all_tangos_features)\n    })\n    \n    # Save OOF predictions\n    np.save(PREDICTIONS_DIR / \"tangos_oof.npy\", oof_preds)\n    \n    return test_predictions, oof_preds\n\n# =========================\n# DOFEN Training\n# =========================\ndef train_dofen(train_df, test_df):\n    \"\"\"Train DOFEN model\"\"\"\n    print(\"\\n=== Training DOFEN Model ===\")\n    print(\"Deep Oblivious Forest ENsemble - State-of-the-art for tabular data\")\n    \n    # Hyperparameters\n    hparams = {\n        \"seed\": 42,\n        \"num_epochs\": 20,\n        \"batch_size\": 512,\n        \"learning_rate\": 0.0005,\n        \"weight_decay\": 1e-4,\n        \"num_layers\": 2,\n        \"trees_per_layer\": [50, 25],\n        \"tree_depth\": 4,\n        \"feature_subset_ratio\": 0.6,\n        \"temperature\": 1.5,\n        \"use_sparse_selection\": True,\n        \"dropout\": 0.2,\n        \"gradient_clip\": 0.5,\n        \"patience\": 5,\n        \"warmup_epochs\": 3\n    }\n    \n    set_seed(hparams[\"seed\"])\n    \n    # Use GANDALF features plus some additional ones for DOFEN\n    dofen_features = Config.GANDALF_FEATURES.copy()\n    \n    # Add specific engineered features that work well with trees\n    tree_friendly_features = [\n        \"log_volume\", \"net_order_flow\", \"normalized_net_flow\",\n        \"total_depth\", \"depth_imbalance\", \"kyle_lambda\",\n        \"volume_depth_ratio\", \"price_impact_proxy\",\n        \"order_flow_imbalance\", \"bid_ask_imbalance\",\n        \"liquidity_ratio\", \"market_stress\", \n        \"trade_informativeness\", \"execution_rate\",\n        \"buy_sell_ratio\", \"selling_pressure\"\n    ]\n    \n    all_dofen_features = dofen_features + tree_friendly_features\n    all_dofen_features = list(set(all_dofen_features))\n    all_dofen_features = [f for f in all_dofen_features if f in train_df.columns]\n    \n    print(f\"Using {len(all_dofen_features)} features for DOFEN\")\n    \n    # Prepare data\n    X_train_full = train_df[all_dofen_features].values\n    y_train_full = train_df[Config.LABEL_COLUMN].values\n    \n    # Keep track of OOF predictions\n    oof_preds = np.zeros(len(train_df))\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    # Collect all fold predictions\n    test_preds_all_folds = []\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train_full), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        \n        X_train = X_train_full[train_idx]\n        y_train = y_train_full[train_idx]\n        X_val = X_train_full[valid_idx]\n        y_val = y_train_full[valid_idx]\n        \n        # Use QuantileTransformer for tree-based models\n        scaler = QuantileTransformer(\n            n_quantiles=min(1000, X_train.shape[0]), \n            output_distribution='uniform',\n            random_state=42\n        )\n        X_train = scaler.fit_transform(X_train)\n        X_val = scaler.transform(X_val)\n        X_test = scaler.transform(test_df[all_dofen_features].values)\n        \n        # Handle any NaN values\n        X_train = np.nan_to_num(X_train, nan=0.0, posinf=1.0, neginf=0.0)\n        X_val = np.nan_to_num(X_val, nan=0.0, posinf=1.0, neginf=0.0)\n        X_test = np.nan_to_num(X_test, nan=0.0, posinf=1.0, neginf=0.0)\n        \n        # Create dataloaders\n        train_loader = get_dataloaders(X_train, y_train, hparams, device, shuffle=True)\n        val_loader = get_dataloaders(X_val, y_val, hparams, device, shuffle=False)\n        test_loader = get_dataloaders(X_test, None, hparams, device, shuffle=False)\n        \n        # Initialize model\n        model = DOFEN(\n            num_features=len(all_dofen_features),\n            num_layers=hparams[\"num_layers\"],\n            trees_per_layer=hparams[\"trees_per_layer\"],\n            tree_depth=hparams[\"tree_depth\"],\n            feature_subset_ratio=hparams[\"feature_subset_ratio\"],\n            temperature=hparams[\"temperature\"],\n            use_sparse_selection=hparams[\"use_sparse_selection\"],\n            dropout=hparams[\"dropout\"]\n        ).to(device)\n        \n        # Loss and optimizer\n        criterion = nn.HuberLoss(delta=1.0, reduction='mean')\n        optimizer = optim.AdamW(\n            model.parameters(), \n            lr=hparams[\"learning_rate\"],\n            weight_decay=hparams[\"weight_decay\"]\n        )\n        \n        # Learning rate scheduler with warmup\n        def lr_lambda(epoch):\n            if epoch < hparams[\"warmup_epochs\"]:\n                return (epoch + 1) / hparams[\"warmup_epochs\"]\n            else:\n                # Cosine annealing after warmup\n                progress = (epoch - hparams[\"warmup_epochs\"]) / (hparams[\"num_epochs\"] - hparams[\"warmup_epochs\"])\n                return 0.5 * (1 + math.cos(math.pi * progress))\n        \n        scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)\n        \n        checkpoint_path = CHECKPOINTS_DIR / f\"dofen_fold{fold}_model.pt\"\n        best_pearson = -np.inf\n        patience_counter = 0\n        \n        # Training loop\n        num_epochs = hparams[\"num_epochs\"]\n        \n        for epoch in range(num_epochs):\n            model.train()\n            running_loss = 0.0\n            \n            for inputs, targets in train_loader:\n                inputs, targets = inputs.to(device), targets.to(device)\n                \n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, targets)\n                \n                # Check for NaN\n                if torch.isnan(loss):\n                    print(f\"NaN loss detected at epoch {epoch+1}, skipping batch\")\n                    continue\n                \n                loss.backward()\n                \n                # Gradient clipping\n                torch.nn.utils.clip_grad_norm_(model.parameters(), hparams[\"gradient_clip\"])\n                \n                optimizer.step()\n                \n                running_loss += loss.item() * inputs.size(0)\n            \n            scheduler.step()\n            \n            running_loss = running_loss / len(train_loader.dataset)\n            \n            # Validation\n            model.eval()\n            val_loss = 0.0\n            preds = []\n            trues = []\n            \n            with torch.no_grad():\n                for inputs, targets in val_loader:\n                    inputs, targets = inputs.to(device), targets.to(device)\n                    outputs = model(inputs)\n                    loss = criterion(outputs, targets)\n                    \n                    if not torch.isnan(loss):\n                        val_loss += loss.item() * inputs.size(0)\n                        preds.extend(outputs.cpu().numpy())\n                        trues.extend(targets.cpu().numpy())\n            \n            val_loss /= len(val_loader.dataset)\n            \n            if len(preds) > 0:\n                preds = np.array(preds)\n                trues = np.array(trues)\n                pearson_coef = pearsonr(preds, trues)[0]\n            else:\n                pearson_coef = -np.inf\n            \n            # Save best model\n            if pearson_coef > best_pearson:\n                best_pearson = pearson_coef\n                torch.save(model.state_dict(), checkpoint_path)\n                patience_counter = 0\n            else:\n                patience_counter += 1\n                if patience_counter >= hparams[\"patience\"]:\n                    break\n        \n        # Load best model and save OOF predictions\n        model.load_state_dict(torch.load(checkpoint_path))\n        model.eval()\n        \n        val_preds = []\n        with torch.no_grad():\n            for inputs, targets in val_loader:\n                inputs = inputs.to(device)\n                outputs = model(inputs)\n                val_preds.extend(outputs.cpu().numpy())\n        \n        oof_preds[valid_idx] = np.array(val_preds)\n        \n        # Make test predictions\n        predictions = []\n        with torch.no_grad():\n            for inputs in test_loader:\n                inputs = inputs[0].to(device)\n                outputs = model(inputs)\n                predictions.extend(outputs.cpu().numpy())\n        \n        test_preds_all_folds.append(np.array(predictions))\n    \n    # Average test predictions across folds\n    test_predictions = np.mean(test_preds_all_folds, axis=0)\n    \n    # Post-process predictions\n    pred_mean = train_df[Config.LABEL_COLUMN].mean()\n    pred_std = train_df[Config.LABEL_COLUMN].std()\n    test_predictions = np.clip(test_predictions, pred_mean - 4 * pred_std, pred_mean + 4 * pred_std)\n    \n    # Calculate overall OOF score\n    oof_score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds)[0]\n    print(f\"\\nDOFEN Overall OOF Pearson: {oof_score:.4f}\")\n    \n    # Save predictions\n    save_predictions(\"dofen\", test_predictions, {\n        \"oof_score\": oof_score,\n        \"n_features\": len(all_dofen_features),\n        \"n_layers\": hparams[\"num_layers\"],\n        \"trees_per_layer\": hparams[\"trees_per_layer\"],\n        \"tree_depth\": hparams[\"tree_depth\"]\n    })\n    \n    # Save OOF predictions\n    np.save(PREDICTIONS_DIR / \"dofen_oof.npy\", oof_preds)\n    \n    return test_predictions, oof_preds\n\n# =========================\n# Ensemble Analysis Functions\n# =========================\ndef optimize_ensemble_weights(predictions_dict, y_true, method='scipy'):\n    \"\"\"Optimize ensemble weights using various methods\"\"\"\n    \n    if method == 'scipy':\n        # Use scipy optimization\n        def objective(weights):\n            weights = weights / weights.sum()\n            ensemble = sum(w * pred for w, pred in zip(weights, predictions_dict.values()))\n            return -pearsonr(y_true, ensemble)[0]\n        \n        n_models = len(predictions_dict)\n        initial_weights = np.ones(n_models) / n_models\n        bounds = [(0, 1) for _ in range(n_models)]\n        \n        result = minimize(objective, initial_weights, bounds=bounds, method='SLSQP')\n        optimal_weights = result.x / result.x.sum()\n        \n    elif method == 'greedy':\n        # Greedy forward selection\n        model_names = list(predictions_dict.keys())\n        selected_models = []\n        remaining_models = model_names.copy()\n        weights = []\n        \n        while remaining_models:\n            best_score = -np.inf\n            best_model = None\n            best_weight = None\n            \n            for model in remaining_models:\n                # Try different weights for the new model\n                for w in np.linspace(0.1, 0.9, 9):\n                    if selected_models:\n                        # Combine with existing ensemble\n                        existing_weight = 1 - w\n                        existing_weights = np.array(weights) * existing_weight / sum(weights) if sum(weights) > 0 else np.array(weights)\n                        \n                        ensemble = sum(existing_weights[i] * predictions_dict[selected_models[i]] \n                                     for i in range(len(selected_models)))\n                        ensemble += w * predictions_dict[model]\n                    else:\n                        ensemble = predictions_dict[model]\n                    \n                    score = pearsonr(y_true, ensemble)[0]\n                    if score > best_score:\n                        best_score = score\n                        best_model = model\n                        best_weight = w\n            \n            if best_model:\n                selected_models.append(best_model)\n                weights.append(best_weight)\n                remaining_models.remove(best_model)\n                \n                # Normalize weights\n                weights = list(np.array(weights) / sum(weights))\n                \n                print(f\"Added {best_model} with weight {best_weight:.3f}, score: {best_score:.4f}\")\n        \n        # Create weight dict\n        optimal_weights = np.zeros(len(model_names))\n        for i, model in enumerate(model_names):\n            if model in selected_models:\n                idx = selected_models.index(model)\n                optimal_weights[i] = weights[idx]\n    \n    else:\n        # Simple average\n        n_models = len(predictions_dict)\n        optimal_weights = np.ones(n_models) / n_models\n    \n    return optimal_weights\n\ndef analyze_model_correlations(predictions_dict, save_path=None):\n    \"\"\"Analyze correlations between model predictions\"\"\"\n    model_names = list(predictions_dict.keys())\n    n_models = len(model_names)\n    \n    # Create correlation matrix\n    corr_matrix = np.zeros((n_models, n_models))\n    for i, model1 in enumerate(model_names):\n        for j, model2 in enumerate(model_names):\n            corr_matrix[i, j] = np.corrcoef(predictions_dict[model1], \n                                           predictions_dict[model2])[0, 1]\n    \n    # Print correlation matrix\n    print(\"\\n📊 Model Correlation Matrix:\")\n    print(\"       \", \"  \".join([m[:6] for m in model_names]))\n    for i, model in enumerate(model_names):\n        print(f\"{model[:6]:6s}\", \" \".join([f\"{corr_matrix[i, j]:6.3f}\" for j in range(n_models)]))\n    \n    if save_path:\n        np.save(save_path, corr_matrix)\n    \n    return corr_matrix\n\ndef create_ensemble_combinations(predictions_dict, y_true, submission_df):\n    \"\"\"Create and evaluate different ensemble combinations\"\"\"\n    \n    model_names = list(predictions_dict.keys())\n    results = []\n    \n    # 1. Individual models\n    print(\"\\n=== Individual Model Scores ===\")\n    for model in model_names:\n        score = pearsonr(y_true, predictions_dict[model])[0]\n        results.append({\n            'ensemble_name': model,\n            'models': [model],\n            'weights': {model: 1.0},\n            'oof_score': score,\n            'n_models': 1\n        })\n        print(f\"{model}: {score:.4f}\")\n    \n    # 2. All possible pairs\n    print(\"\\n=== Best Model Pairs ===\")\n    pair_results = []\n    for i, model1 in enumerate(model_names):\n        for j, model2 in enumerate(model_names[i+1:], i+1):\n            # Try different weight combinations\n            best_score = -np.inf\n            best_w1 = 0.5\n            \n            for w1 in np.linspace(0.1, 0.9, 9):\n                w2 = 1 - w1\n                ensemble = w1 * predictions_dict[model1] + w2 * predictions_dict[model2]\n                score = pearsonr(y_true, ensemble)[0]\n                \n                if score > best_score:\n                    best_score = score\n                    best_w1 = w1\n            \n            pair_results.append({\n                'ensemble_name': f\"{model1}_{model2}\",\n                'models': [model1, model2],\n                'weights': {model1: best_w1, model2: 1-best_w1},\n                'oof_score': best_score,\n                'n_models': 2\n            })\n    \n    # Sort and print top pairs\n    pair_results.sort(key=lambda x: x['oof_score'], reverse=True)\n    for result in pair_results[:5]:\n        models = result['models']\n        weights = result['weights']\n        print(f\"{models[0]}({weights[models[0]]:.2f}) + {models[1]}({weights[models[1]]:.2f}): {result['oof_score']:.4f}\")\n    \n    results.extend(pair_results)\n    \n    # 3. Top 3, 4, 5 models\n    print(\"\\n=== Multi-Model Ensembles ===\")\n    \n    # Sort models by individual performance\n    model_scores = [(model, pearsonr(y_true, predictions_dict[model])[0]) for model in model_names]\n    model_scores.sort(key=lambda x: x[1], reverse=True)\n    top_models = [m[0] for m in model_scores]\n    \n    for n in [3, 4, 5, 6, 7, 8]:\n        if n > len(model_names):\n            break\n            \n        selected_models = top_models[:n]\n        selected_preds = {m: predictions_dict[m] for m in selected_models}\n        \n        # Optimize weights\n        optimal_weights = optimize_ensemble_weights(selected_preds, y_true, method='scipy')\n        \n        # Create ensemble\n        ensemble = sum(w * selected_preds[m] for w, m in zip(optimal_weights, selected_models))\n        score = pearsonr(y_true, ensemble)[0]\n        \n        weight_dict = {m: w for m, w in zip(selected_models, optimal_weights)}\n        \n        results.append({\n            'ensemble_name': f\"top_{n}_models\",\n            'models': selected_models,\n            'weights': weight_dict,\n            'oof_score': score,\n            'n_models': n\n        })\n        \n        print(f\"Top {n} models: {score:.4f}\")\n        for m, w in weight_dict.items():\n            if w > 0.01:\n                print(f\"  {m}: {w:.3f}\")\n    \n    # 4. Custom ensembles based on model types\n    print(\"\\n=== Custom Ensembles ===\")\n    \n    # Tree-based ensemble\n    tree_models = ['xgboost', 'gandalf', 'simplified_gandalf', 'dofen']\n    tree_models = [m for m in tree_models if m in model_names]\n    if len(tree_models) > 1:\n        tree_preds = {m: predictions_dict[m] for m in tree_models}\n        tree_weights = optimize_ensemble_weights(tree_preds, y_true, method='scipy')\n        tree_ensemble = sum(w * tree_preds[m] for w, m in zip(tree_weights, tree_models))\n        tree_score = pearsonr(y_true, tree_ensemble)[0]\n        \n        results.append({\n            'ensemble_name': 'tree_based_ensemble',\n            'models': tree_models,\n            'weights': {m: w for m, w in zip(tree_models, tree_weights)},\n            'oof_score': tree_score,\n            'n_models': len(tree_models)\n        })\n        print(f\"Tree-based ensemble: {tree_score:.4f}\")\n    \n    # Neural network ensemble\n    nn_models = ['mlp', 'dcnv2', 'anam', 'tangos']\n    nn_models = [m for m in nn_models if m in model_names]\n    if len(nn_models) > 1:\n        nn_preds = {m: predictions_dict[m] for m in nn_models}\n        nn_weights = optimize_ensemble_weights(nn_preds, y_true, method='scipy')\n        nn_ensemble = sum(w * nn_preds[m] for w, m in zip(nn_weights, nn_models))\n        nn_score = pearsonr(y_true, nn_ensemble)[0]\n        \n        results.append({\n            'ensemble_name': 'neural_network_ensemble',\n            'models': nn_models,\n            'weights': {m: w for m, w in zip(nn_models, nn_weights)},\n            'oof_score': nn_score,\n            'n_models': len(nn_models)\n        })\n        print(f\"Neural network ensemble: {nn_score:.4f}\")\n    \n    # 5. Greedy ensemble\n    print(\"\\n=== Greedy Forward Selection ===\")\n    greedy_weights = optimize_ensemble_weights(predictions_dict, y_true, method='greedy')\n    greedy_ensemble = sum(w * predictions_dict[m] for w, m in zip(greedy_weights, model_names))\n    greedy_score = pearsonr(y_true, greedy_ensemble)[0]\n    \n    results.append({\n        'ensemble_name': 'greedy_selection',\n        'models': model_names,\n        'weights': {m: w for m, w in zip(model_names, greedy_weights)},\n        'oof_score': greedy_score,\n        'n_models': sum(1 for w in greedy_weights if w > 0.01)\n    })\n    \n    # Sort all results by score\n    results.sort(key=lambda x: x['oof_score'], reverse=True)\n    \n    # Save results\n    results_df = pd.DataFrame(results)\n    results_df.to_csv(ENSEMBLE_DIR / \"ensemble_analysis.csv\", index=False)\n    \n    # Save detailed results\n    with open(ENSEMBLE_DIR / \"ensemble_details.json\", 'w') as f:\n        json.dump(results, f, indent=4)\n    \n    return results\n\ndef create_submission(predictions, submission_df, name):\n    \"\"\"Create submission file\"\"\"\n    submission = submission_df.copy()\n    submission[\"prediction\"] = predictions\n    submission_path = SUBMISSIONS_DIR / f\"{name}.csv\"\n    submission.to_csv(submission_path, index=False)\n    return submission_path\n\n# =========================\n# Main Execution\n# =========================\ndef main():\n    \"\"\"Main execution function\"\"\"\n    \n    # Load data\n    print(\"\\n=== Loading Data ===\")\n    train_df, test_df, submission_df = load_data()\n    \n    # Dictionary to store all predictions\n    test_predictions = {}\n    oof_predictions = {}\n    \n    # Train all models\n    print(\"\\n=== Training All Models ===\")\n    \n    # XGBoost\n    xgb_test, xgb_oof = train_xgboost(train_df, test_df)\n    test_predictions['xgboost'] = xgb_test\n    oof_predictions['xgboost'] = xgb_oof\n    \n    # MLP\n    mlp_test, mlp_oof = train_mlp(train_df, test_df)\n    test_predictions['mlp'] = mlp_test\n    oof_predictions['mlp'] = mlp_oof\n    \n    # DCN V2\n    dcnv2_test, dcnv2_oof = train_dcnv2(train_df, test_df)\n    test_predictions['dcnv2'] = dcnv2_test\n    oof_predictions['dcnv2'] = dcnv2_oof\n    \n    # GANDALF models\n    gandalf_test, simplified_test, gandalf_oof, simplified_oof = train_gandalf(train_df, test_df)\n    test_predictions['gandalf'] = gandalf_test\n    test_predictions['simplified_gandalf'] = simplified_test\n    oof_predictions['gandalf'] = gandalf_oof\n    oof_predictions['simplified_gandalf'] = simplified_oof\n    \n    # ANAM\n    anam_test, anam_oof = train_anam(train_df, test_df)\n    test_predictions['anam'] = anam_test\n    oof_predictions['anam'] = anam_oof\n    \n    # TANGOS\n    tangos_test, tangos_oof = train_tangos(train_df, test_df)\n    test_predictions['tangos'] = tangos_test\n    oof_predictions['tangos'] = tangos_oof\n    \n    # DOFEN\n    # dofen_test, dofen_oof = train_dofen(train_df, test_df)\n    # test_predictions['dofen'] = dofen_test\n    # oof_predictions['dofen'] = dofen_oof\n    \n    # Analyze model correlations\n    print(\"\\n=== Analyzing Model Correlations ===\")\n    corr_matrix = analyze_model_correlations(oof_predictions, \n                                           ENSEMBLE_DIR / \"model_correlations.npy\")\n    \n    # Create and evaluate ensemble combinations\n    print(\"\\n=== Creating Ensemble Combinations ===\")\n    y_true = train_df[Config.LABEL_COLUMN].values\n    ensemble_results = create_ensemble_combinations(oof_predictions, y_true, submission_df)\n    \n    # Create submissions for top ensembles\n    print(\"\\n=== Creating Submissions for Top Ensembles ===\")\n    \n    top_n = 5\n    for i, result in enumerate(ensemble_results[:top_n]):\n        ensemble_name = result['ensemble_name']\n        models = result['models']\n        weights = result['weights']\n        oof_score = result['oof_score']\n        \n        print(f\"\\n{i+1}. {ensemble_name} (OOF: {oof_score:.4f})\")\n        \n        # Create test ensemble\n        test_ensemble = np.zeros_like(test_predictions[models[0]])\n        for model in models:\n            if weights[model] > 0.01:  # Only include models with significant weight\n                test_ensemble += weights[model] * test_predictions[model]\n                print(f\"   {model}: {weights[model]:.3f}\")\n        \n        # Create submission\n        submission_path = create_submission(test_ensemble, submission_df, \n                                          f\"ensemble_{i+1}_{ensemble_name}\")\n        print(f\"   Saved to: {submission_path}\")\n    \n    # Print final recommendations\n    print(\"\\n\" + \"=\"*60)\n    print(\"ENSEMBLE RECOMMENDATIONS\")\n    print(\"=\"*60)\n    \n    print(\"\\n🏆 TOP 3 ENSEMBLES TO SUBMIT:\")\n    for i, result in enumerate(ensemble_results[:3]):\n        print(f\"\\n{i+1}. {result['ensemble_name']}\")\n        print(f\"   OOF Score: {result['oof_score']:.4f}\")\n        print(f\"   Models: {', '.join(result['models'])}\")\n        print(f\"   Key weights:\")\n        for model, weight in result['weights'].items():\n            if weight > 0.05:\n                print(f\"     - {model}: {weight:.1%}\")\n    \n    print(\"\\n📊 INSIGHTS:\")\n    print(f\"• Best individual model: {ensemble_results[0]['ensemble_name']} ({ensemble_results[0]['oof_score']:.4f})\")\n    \n    # Find best ensemble\n    best_ensemble = max([r for r in ensemble_results if r['n_models'] > 1], \n                       key=lambda x: x['oof_score'])\n    print(f\"• Best ensemble: {best_ensemble['ensemble_name']} ({best_ensemble['oof_score']:.4f})\")\n    print(f\"• Ensemble improvement: {(best_ensemble['oof_score'] - ensemble_results[0]['oof_score'])*100:.2f}%\")\n    \n    # Model diversity\n    avg_corr = (corr_matrix.sum() - len(corr_matrix)) / (len(corr_matrix) * (len(corr_matrix) - 1))\n    print(f\"• Average model correlation: {avg_corr:.3f}\")\n    \n    print(\"\\n✅ Pipeline execution completed successfully!\")\n    print(f\"📂 All results saved to: {ENSEMBLE_DIR}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}