{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Credit to the following authors and notebooks/discussions:\n* https://www.kaggle.com/code/bakuer30/drw-remix-vi - For tuning and improving XGBoost parameters and features\n* https://www.kaggle.com/competitions/drw-crypto-market-prediction/discussion/581193 - For the idea of temporal weights / time slices\n* https://www.kaggle.com/competitions/drw-crypto-market-prediction/discussion/584475 - For poiting out that the data at the very beginning may be more valuable too","metadata":{}},{"cell_type":"code","source":"# XGBoost + Neural Models Ensemble with Time Slicing\n# ===================================================\n# Keeps the highly-tuned XGBoost model intact and adds complementary neural models\n\n# Configuration - Change this value\nEARLY_PERCENTAGE = 0.35  # Change this to 0.20, 0.25, 0.30, 0.35, 0.40, or 0.45\n\n# Imports\nimport sys\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, QuantileTransformer\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set device and seed\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\ndef set_seed(seed=42):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n\nset_seed(42)\n\n# ====================================\n# Original Feature Engineering (Keep Exactly)\n# ====================================\ndef feature_engineering(df):\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    \n    df = df.replace([np.inf, -np.inf], np.nan)\n    df = df.fillna(0)\n    return df \n\n# ====================================\n# Configuration (Keep Original)\n# ====================================\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\", \"X292\",\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n\n# Original XGBoost parameters (DO NOT CHANGE)\nXGB_PARAMS = {\n    'tree_method': 'hist', \n    'device': 'gpu',\n    'n_jobs': -1,\n    'colsample_bytree': 0.4111224922845363, \n    'colsample_bynode': 0.28869302181383194,\n    'gamma': 1.4665430311056709, \n    'learning_rate': 0.014053505540364681, \n    'max_depth': 7, \n    'max_leaves': 40, \n    'n_estimators': 500,\n    'reg_alpha': 27.791606770656145, \n    'reg_lambda': 84.90603428439086,\n    'subsample': 0.06567,\n    'verbosity': 0,\n    'random_state': Config.RANDOM_STATE\n}\n\nLEARNERS = [\n    {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS},\n]\n\n# ====================================\n# Neural Network Models\n# ====================================\n\n# 1. MLP with Heavy Regularization\nclass RegularizedMLP(nn.Module):\n    def __init__(self, input_dim, hidden_dims=[256, 128, 64], dropout=0.6, noise_factor=0.1):\n        super().__init__()\n        self.noise_factor = noise_factor\n        \n        layers = []\n        prev_dim = input_dim\n        \n        for dim in hidden_dims:\n            layers.extend([\n                nn.Linear(prev_dim, dim),\n                nn.BatchNorm1d(dim),\n                nn.ReLU(),\n                nn.Dropout(dropout)\n            ])\n            prev_dim = dim\n        \n        layers.append(nn.Linear(prev_dim, 1))\n        self.network = nn.Sequential(*layers)\n        \n    def forward(self, x):\n        # Add noise during training\n        if self.training and self.noise_factor > 0:\n            noise = torch.randn_like(x) * self.noise_factor\n            x = x + noise\n        \n        return self.network(x)\n\n# 2. SAINT (Self-Attention and Intersample Attention Transformer)\nclass SAINT(nn.Module):\n    def __init__(self, input_dim, n_heads=8, n_layers=3, d_model=128, dropout=0.3):\n        super().__init__()\n        self.input_dim = input_dim\n        self.d_model = d_model\n        \n        # Input projection\n        self.input_projection = nn.Linear(input_dim, d_model)\n        \n        # Positional encoding\n        self.pos_encoding = nn.Parameter(torch.randn(1, 100, d_model) * 0.02)\n        \n        # Transformer layers\n        self.layers = nn.ModuleList([\n            nn.TransformerEncoderLayer(\n                d_model=d_model,\n                nhead=n_heads,\n                dim_feedforward=d_model * 4,\n                dropout=dropout,\n                batch_first=True\n            ) for _ in range(n_layers)\n        ])\n        \n        # Output head\n        self.output_head = nn.Sequential(\n            nn.LayerNorm(d_model),\n            nn.Linear(d_model, d_model // 2),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(d_model // 2, 1)\n        )\n        \n    def forward(self, x):\n        batch_size = x.shape[0]\n        \n        # Project input\n        x = self.input_projection(x)\n        x = x.unsqueeze(1)  # Add sequence dimension\n        \n        # Add positional encoding\n        x = x + self.pos_encoding[:, :1, :]\n        \n        # Apply transformer layers\n        for layer in self.layers:\n            x = layer(x)\n        \n        # Global pooling\n        x = x.mean(dim=1)\n        \n        # Output\n        return self.output_head(x)\n\n# 3. GANDALF (Simplified version)\nclass GANDALF(nn.Module):\n    def __init__(self, input_dim, n_estimators=20, tree_dim=64, depth=3):\n        super().__init__()\n        self.n_estimators = n_estimators\n        \n        # Feature transformation\n        self.feature_transform = nn.Sequential(\n            nn.Linear(input_dim, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, tree_dim),\n            nn.BatchNorm1d(tree_dim)\n        )\n        \n        # Soft decision trees\n        self.trees = nn.ModuleList([\n            nn.Sequential(\n                nn.Linear(tree_dim, 32),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(32, 16),\n                nn.ReLU(),\n                nn.Linear(16, 1)\n            ) for _ in range(n_estimators)\n        ])\n        \n        # Gating network\n        self.gate = nn.Sequential(\n            nn.Linear(input_dim, 64),\n            nn.ReLU(),\n            nn.Linear(64, n_estimators),\n            nn.Softmax(dim=1)\n        )\n        \n    def forward(self, x):\n        # Transform features\n        tree_input = self.feature_transform(x)\n        \n        # Get tree outputs\n        tree_outputs = []\n        for tree in self.trees:\n            output = tree(tree_input)\n            tree_outputs.append(output)\n        \n        tree_outputs = torch.cat(tree_outputs, dim=1)\n        \n        # Get gates\n        gates = self.gate(x)\n        \n        # Weighted combination\n        output = (tree_outputs * gates).sum(dim=1, keepdim=True)\n        \n        return output\n\n# 4. Additive Model\nclass AdditiveModel(nn.Module):\n    def __init__(self, input_dim, hidden_dim=64):\n        super().__init__()\n        \n        # Shape function for each feature\n        self.shape_functions = nn.ModuleList([\n            nn.Sequential(\n                nn.Linear(1, hidden_dim),\n                nn.ReLU(),\n                nn.Linear(hidden_dim, hidden_dim),\n                nn.ReLU(),\n                nn.Linear(hidden_dim, 1)\n            ) for _ in range(input_dim)\n        ])\n        \n        # Global bias\n        self.bias = nn.Parameter(torch.zeros(1))\n        \n        # Feature importance weights\n        self.feature_weights = nn.Parameter(torch.ones(input_dim))\n        \n    def forward(self, x):\n        outputs = []\n        \n        for i in range(x.shape[1]):\n            feature = x[:, i:i+1]\n            shape_output = self.shape_functions[i](feature)\n            weighted_output = shape_output * torch.sigmoid(self.feature_weights[i])\n            outputs.append(weighted_output)\n        \n        # Sum all shape functions\n        output = sum(outputs) + self.bias\n        \n        return output\n\n# ====================================\n# Data Loading (Keep Original)\n# ====================================\ndef create_time_decay_weights(n: int, decay: float = 0.9, reverse: bool = False) -> np.ndarray:\n    \"\"\"Create time decay weights. If reverse=True, older data gets higher weight.\"\"\"\n    positions = np.arange(n)\n    if reverse:\n        normalized = 1.0 - (positions / (n - 1))\n    else:\n        normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef load_data():\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n\n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    \n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\nConfig.FEATURES += [\"bid_qty\", \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\"]\nConfig.FEATURES = list(set(Config.FEATURES))\n\n# ====================================\n# Original XGBoost Training (Keep Exactly)\n# ====================================\ndef get_model_slices(n_samples: int):\n    return [\n        {\"name\": \"full_data\", \"type\": \"full\", \"cutoff\": 0},\n        {\"name\": \"last_75pct\", \"type\": \"recent\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_50pct\", \"type\": \"recent\", \"cutoff\": int(0.50 * n_samples)},\n        {\"name\": f\"first_{int(EARLY_PERCENTAGE*100)}pct\", \"type\": \"early\", \"cutoff\": int(EARLY_PERCENTAGE * n_samples)},\n    ]\n\ndef train_xgboost(train_df, test_df):\n    \"\"\"Original XGBoost training function - DO NOT MODIFY\"\"\"\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS\n    }\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            slice_type = s[\"type\"]\n            \n            if slice_type == \"full\":\n                subset = train_df.reset_index(drop=True)\n                rel_idx = train_idx\n                sw = full_weights[train_idx]\n            elif slice_type == \"recent\":\n                subset = train_df.iloc[cutoff:].reset_index(drop=True)\n                rel_idx = train_idx[train_idx >= cutoff] - cutoff\n                if cutoff > 0:\n                    sw = create_time_decay_weights(len(subset))[rel_idx]\n                else:\n                    sw = full_weights[train_idx]\n            elif slice_type == \"early\":\n                subset = train_df.iloc[:cutoff].reset_index(drop=True)\n                rel_idx = train_idx[train_idx < cutoff]\n                if len(rel_idx) > 0:\n                    sw = create_time_decay_weights(len(subset))[rel_idx]\n                else:\n                    sw = np.array([])\n\n            if len(rel_idx) == 0:\n                print(f\"  Skipping slice: {slice_name} (no training data in fold)\")\n                continue\n\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            \n            X_train_np = X_train.values\n            y_train_np = y_train.values\n            X_valid_np = X_valid.values\n            y_valid_np = y_valid.values\n            \n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n\n            for learner in LEARNERS:\n                model = learner[\"Estimator\"](**learner[\"params\"])\n                model.fit(X_train_np, y_train_np, sample_weight=sw, \n                          eval_set=[(X_valid_np, y_valid_np)], verbose=False)\n                \n                if slice_type == \"early\":\n                    mask = valid_idx < cutoff\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n                    if (~mask).any():\n                        oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n                else:\n                    mask = valid_idx >= cutoff if slice_type == \"recent\" else np.ones(len(valid_idx), dtype=bool)\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n                    if slice_type == \"recent\" and cutoff > 0 and (~mask).any():\n                        oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][valid_idx[~mask]]\n\n                test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[Config.FEATURES])\n\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= Config.N_FOLDS\n\n    return oof_preds, test_preds, model_slices\n\n# ====================================\n# Neural Network Training\n# ====================================\ndef train_neural_model(model, train_loader, val_loader, epochs=30, lr=0.001, patience=5):\n    \"\"\"Generic training function for neural models\"\"\"\n    optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.5)\n    criterion = nn.HuberLoss(delta=1.0)\n    \n    best_val_loss = float('inf')\n    patience_counter = 0\n    best_state = None\n    \n    for epoch in range(epochs):\n        # Training\n        model.train()\n        train_loss = 0\n        for batch_x, batch_y in train_loader:\n            batch_x = batch_x.to(device)\n            batch_y = batch_y.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(batch_x)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            optimizer.step()\n            \n            train_loss += loss.item()\n        \n        # Validation\n        model.eval()\n        val_loss = 0\n        val_preds = []\n        val_targets = []\n        \n        with torch.no_grad():\n            for batch_x, batch_y in val_loader:\n                batch_x = batch_x.to(device)\n                batch_y = batch_y.to(device)\n                \n                outputs = model(batch_x)\n                loss = criterion(outputs, batch_y)\n                val_loss += loss.item()\n                \n                val_preds.extend(outputs.cpu().numpy().flatten())\n                val_targets.extend(batch_y.cpu().numpy().flatten())\n        \n        avg_val_loss = val_loss / len(val_loader)\n        scheduler.step(avg_val_loss)\n        \n        if avg_val_loss < best_val_loss:\n            best_val_loss = avg_val_loss\n            best_state = model.state_dict().copy()\n            patience_counter = 0\n        else:\n            patience_counter += 1\n            if patience_counter >= patience:\n                break\n    \n    # Load best model\n    if best_state is not None:\n        model.load_state_dict(best_state)\n    \n    return model\n\ndef train_neural_models(train_df, test_df):\n    \"\"\"Train all neural network models\"\"\"\n    print(\"\\n=== Training Neural Network Models ===\")\n    \n    X_train = train_df[Config.FEATURES].values\n    y_train = train_df[Config.LABEL_COLUMN].values\n    X_test = test_df[Config.FEATURES].values\n    \n    # Models to train\n    models = {\n        'mlp': RegularizedMLP(len(Config.FEATURES), hidden_dims=[256, 128, 64], dropout=0.6, noise_factor=0.1),\n        'saint': SAINT(len(Config.FEATURES), n_heads=8, n_layers=3, d_model=128, dropout=0.3),\n        'gandalf': GANDALF(len(Config.FEATURES), n_estimators=20, tree_dim=64),\n        'additive': AdditiveModel(len(Config.FEATURES), hidden_dim=64)\n    }\n    \n    # Store predictions\n    nn_oof_preds = {name: np.zeros(len(train_df)) for name in models}\n    nn_test_preds = {name: np.zeros(len(test_df)) for name in models}\n    \n    # K-fold training\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    for model_name, model_class in models.items():\n        print(f\"\\nTraining {model_name.upper()}...\")\n        \n        for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train), 1):\n            print(f\"  Fold {fold}/{Config.N_FOLDS}\")\n            \n            # Prepare data\n            X_tr, X_val = X_train[train_idx], X_train[valid_idx]\n            y_tr, y_val = y_train[train_idx], y_train[valid_idx]\n            \n            # Scale data\n            scaler = StandardScaler()\n            X_tr_scaled = scaler.fit_transform(X_tr)\n            X_val_scaled = scaler.transform(X_val)\n            X_test_scaled = scaler.transform(X_test)\n            \n            # Create datasets\n            train_dataset = TensorDataset(\n                torch.tensor(X_tr_scaled, dtype=torch.float32),\n                torch.tensor(y_tr, dtype=torch.float32).unsqueeze(1)\n            )\n            val_dataset = TensorDataset(\n                torch.tensor(X_val_scaled, dtype=torch.float32),\n                torch.tensor(y_val, dtype=torch.float32).unsqueeze(1)\n            )\n            \n            # Adjust batch size based on model\n            batch_size = 256 if model_name in ['mlp', 'additive'] else 128\n            \n            train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n            val_loader = DataLoader(val_dataset, batch_size=batch_size*2, shuffle=False)\n            \n            # Create new model instance\n            if model_name == 'mlp':\n                model = RegularizedMLP(len(Config.FEATURES), hidden_dims=[256, 128, 64], dropout=0.6, noise_factor=0.1)\n            elif model_name == 'saint':\n                model = SAINT(len(Config.FEATURES), n_heads=8, n_layers=3, d_model=128, dropout=0.3)\n            elif model_name == 'gandalf':\n                model = GANDALF(len(Config.FEATURES), n_estimators=20, tree_dim=64)\n            else:  # additive\n                model = AdditiveModel(len(Config.FEATURES), hidden_dim=64)\n            \n            model = model.to(device)\n            \n            # Train model\n            model = train_neural_model(model, train_loader, val_loader, epochs=30, lr=0.001)\n            \n            # Make predictions\n            model.eval()\n            with torch.no_grad():\n                # Validation predictions\n                val_preds = []\n                for batch_x, _ in val_loader:\n                    batch_x = batch_x.to(device)\n                    outputs = model(batch_x)\n                    val_preds.extend(outputs.cpu().numpy().flatten())\n                \n                nn_oof_preds[model_name][valid_idx] = np.array(val_preds)\n                \n                # Test predictions\n                test_dataset = TensorDataset(torch.tensor(X_test_scaled, dtype=torch.float32))\n                test_loader = DataLoader(test_dataset, batch_size=batch_size*2, shuffle=False)\n                \n                test_preds = []\n                for (batch_x,) in test_loader:\n                    batch_x = batch_x.to(device)\n                    outputs = model(batch_x)\n                    test_preds.extend(outputs.cpu().numpy().flatten())\n                \n                nn_test_preds[model_name] += np.array(test_preds) / Config.N_FOLDS\n        \n        # Report OOF score\n        oof_score = pearsonr(y_train, nn_oof_preds[model_name])[0]\n        print(f\"  {model_name.upper()} OOF Score: {oof_score:.4f}\")\n    \n    return nn_oof_preds, nn_test_preds\n\n# ====================================\n# Final Ensemble\n# ====================================\ndef create_final_ensemble(train_df, xgb_oof, xgb_test, nn_oof, nn_test, submission_df):\n    \"\"\"Create final ensemble with XGBoost getting 80% weight\"\"\"\n    print(\"\\n=== Creating Final Ensemble ===\")\n    \n    y_true = train_df[Config.LABEL_COLUMN].values\n    \n    # Evaluate XGBoost slices\n    xgb_scores = {}\n    for slice_name in xgb_oof['xgb']:\n        score = pearsonr(y_true, xgb_oof['xgb'][slice_name])[0]\n        xgb_scores[slice_name] = score\n        print(f\"  XGB {slice_name}: {score:.4f}\")\n    \n    # Find best XGBoost slice\n    best_xgb_slice = max(xgb_scores.items(), key=lambda x: x[1])[0]\n    best_xgb_oof = xgb_oof['xgb'][best_xgb_slice]\n    best_xgb_test = xgb_test['xgb'][best_xgb_slice]\n    print(f\"\\nBest XGBoost slice: {best_xgb_slice} ({xgb_scores[best_xgb_slice]:.4f})\")\n    \n    # Evaluate neural models\n    nn_scores = {}\n    for model_name in nn_oof:\n        score = pearsonr(y_true, nn_oof[model_name])[0]\n        nn_scores[model_name] = score\n        print(f\"  {model_name.upper()}: {score:.4f}\")\n    \n    # Create weighted ensemble\n    # XGBoost gets 80% weight\n    xgb_weight = 0.80\n    remaining_weight = 1.0 - xgb_weight\n    \n    # Distribute remaining 20% among neural models based on performance\n    total_nn_score = sum(nn_scores.values())\n    nn_weights = {name: (score/total_nn_score) * remaining_weight for name, score in nn_scores.items()}\n    \n    print(f\"\\n=== Final Weights ===\")\n    print(f\"XGBoost ({best_xgb_slice}): {xgb_weight:.1%}\")\n    for name, weight in nn_weights.items():\n        print(f\"{name.upper()}: {weight:.1%}\")\n    \n    # Create final predictions\n    final_oof = best_xgb_oof * xgb_weight\n    final_test = best_xgb_test * xgb_weight\n    \n    for model_name, weight in nn_weights.items():\n        final_oof += nn_oof[model_name] * weight\n        final_test += nn_test[model_name] * weight\n    \n    final_score = pearsonr(y_true, final_oof)[0]\n    print(f\"\\nFinal Ensemble OOF Score: {final_score:.4f}\")\n    \n    # Alternative ensemble: Simple average of all models (for comparison)\n    all_oof = [best_xgb_oof] + list(nn_oof.values())\n    simple_oof = np.mean(all_oof, axis=0)\n    simple_score = pearsonr(y_true, simple_oof)[0]\n    print(f\"Simple Average Score: {simple_score:.4f}\")\n    \n    # Use the better ensemble\n    if simple_score > final_score:\n        print(\"\\nUsing simple average ensemble\")\n        final_test = np.mean([best_xgb_test] + list(nn_test.values()), axis=0)\n    \n    # Save submission\n    filename = f\"submission_xgb80_neural20_early_{int(EARLY_PERCENTAGE*100)}pct.csv\"\n    submission_df[\"prediction\"] = final_test\n    submission_df.to_csv(filename, index=False)\n    print(f\"\\nSaved: {filename}\")\n    \n    return final_test\n\n# ====================================\n# Main Pipeline\n# ====================================\ndef main():\n    print(f\"\\n{'='*60}\")\n    print(f\"XGBoost (80%) + Neural Models (20%) Ensemble\")\n    print(f\"Running with EARLY_PERCENTAGE = {EARLY_PERCENTAGE} ({int(EARLY_PERCENTAGE*100)}%)\")\n    print(f\"{'='*60}\")\n    \n    # Load data\n    train_df, test_df, submission_df = load_data()\n    \n    # Train original XGBoost (unchanged)\n    print(\"\\n=== Training XGBoost (Original) ===\")\n    xgb_oof, xgb_test, model_slices = train_xgboost(train_df, test_df)\n    \n    # Train neural models\n    nn_oof, nn_test = train_neural_models(train_df, test_df)\n    \n    # Create final ensemble\n    final_predictions = create_final_ensemble(\n        train_df, xgb_oof, xgb_test, nn_oof, nn_test, submission_df\n    )\n    \n    print(\"\\n✅ Pipeline completed successfully!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}