{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### 🙏 Acknowledgements  \n\nthanks to https://www.kaggle.com/code/shinchen93/drw-xgb-train-data-recency\n\nI just add a simple NN, but it seems no improvement.\n\nCredit to all authors of previous notebooks and discussions — your work has been incredibly inspiring!\n\n---\n\n### 🔍 Key Observations\n\nHere are two things that stood out to me during this competition:\n\n1. **Data Length Matters**  \n   Models trained on the full dataset (or at least the most recent 90%) consistently outperform those trained on only the most recent 50%. It seems more historical context helps generalization.\n\n2. **Feature Interactions Can Be Powerful**  \n   Some features, when used jointly (not in isolation!), significantly boost performance. So exploring interactions is well worth the effort.\n\nI hope some of these findings can help you improve your score — good luck!\n\n---\n\n### ❓ Still a Work in Progress\n\nOne challenge I'm still facing:\n\n> I haven’t been able to design a solid cross-validation strategy that replicates the test set correlation reliably.  \n> If you’ve found a way to bridge this gap, I’d love to hear your thoughts!","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nimport numpy as np\nimport pandas as pd\n\n# Neural network imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.preprocessing import StandardScaler\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T16:19:17.250552Z","iopub.execute_input":"2025-06-29T16:19:17.250931Z","iopub.status.idle":"2025-06-29T16:19:18.656329Z","shell.execute_reply.started":"2025-06-29T16:19:17.250895Z","shell.execute_reply":"2025-06-29T16:19:18.655474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Configuration\n# =========================\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\", \"bid_qty\",\n        \"ask_qty\", \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\"X817\", \n        \"X586\",  \"X292\"\n    ]\n\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\nLEARNERS = [\n    {\"name\": \"xgb\", \"Estimator\": XGBRegressor, \"params\": XGB_PARAMS}\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-29T16:19:18.657864Z","iopub.execute_input":"2025-06-29T16:19:18.658391Z","iopub.status.idle":"2025-06-29T16:19:18.665631Z","shell.execute_reply.started":"2025-06-29T16:19:18.658359Z","shell.execute_reply":"2025-06-29T16:19:18.664712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Neural Network Configuration\nclass MetaLearner(nn.Module):\n    \"\"\"Lightweight neural network for meta-learning on refined features\"\"\"\n    def __init__(self, n_features):\n        super(MetaLearner, self).__init__()\n        # Input: refined features + XGBoost prediction\n        input_dim = n_features + 1\n        \n        self.network = nn.Sequential(\n            nn.Linear(input_dim, 64),\n            nn.ReLU(),\n            nn.BatchNorm1d(64),\n            nn.Dropout(0.2),\n            \n            nn.Linear(64, 32),\n            nn.ReLU(),\n            nn.BatchNorm1d(32),\n            nn.Dropout(0.1),\n            \n            nn.Linear(32, 16),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            \n            nn.Linear(16, 1)\n        )\n        \n    def forward(self, x):\n        return self.network(x).squeeze()\n\ndef train_meta_learner(X_train, y_train, xgb_pred_train, X_val, y_val, xgb_pred_val, sample_weights=None):\n    \"\"\"Train meta-learner neural network\"\"\"\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # Prepare features: original features + XGBoost prediction\n    X_train_meta = np.column_stack([X_train, xgb_pred_train])\n    X_val_meta = np.column_stack([X_val, xgb_pred_val])\n    \n    # Standardize features\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train_meta)\n    X_val_scaled = scaler.transform(X_val_meta)\n    \n    # Convert to tensors\n    X_train_tensor = torch.FloatTensor(X_train_scaled).to(device)\n    y_train_tensor = torch.FloatTensor(y_train).to(device)\n    X_val_tensor = torch.FloatTensor(X_val_scaled).to(device)\n    y_val_tensor = torch.FloatTensor(y_val).to(device)\n    \n    # Sample weights\n    if sample_weights is not None:\n        weights_tensor = torch.FloatTensor(sample_weights).to(device)\n    else:\n        weights_tensor = None\n    \n    # Initialize model\n    model = MetaLearner(X_train.shape[1]).to(device)\n    optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)\n    criterion = nn.MSELoss(reduction='none')\n    \n    # Training\n    model.train()\n    best_val_loss = float('inf')\n    patience = 20\n    patience_counter = 0\n    \n    for epoch in range(200):\n        optimizer.zero_grad()\n        \n        # Forward pass\n        pred = model(X_train_tensor)\n        loss = criterion(pred, y_train_tensor)\n        \n        # Apply sample weights if provided\n        if weights_tensor is not None:\n            loss = (loss * weights_tensor).mean()\n        else:\n            loss = loss.mean()\n        \n        # Backward pass\n        loss.backward()\n        optimizer.step()\n        \n        # Validation\n        if epoch % 10 == 0:\n            model.eval()\n            with torch.no_grad():\n                val_pred = model(X_val_tensor)\n                val_loss = criterion(val_pred, y_val_tensor).mean().item()\n                \n                if val_loss < best_val_loss:\n                    best_val_loss = val_loss\n                    patience_counter = 0\n                else:\n                    patience_counter += 1\n                    \n                if patience_counter >= patience:\n                    break\n            model.train()\n    \n    # Final predictions\n    model.eval()\n    with torch.no_grad():\n        train_pred = model(X_train_tensor).cpu().numpy()\n        val_pred = model(X_val_tensor).cpu().numpy()\n    \n    return model, scaler, train_pred, val_pred\n\ndef add_features(df):\n    df['bid_ask_interaction'] = df['bid_qty'] * df['ask_qty']\n    df['bid_buy_interaction'] = df['bid_qty'] * df['buy_qty']\n    df['bid_sell_interaction'] = df['bid_qty'] * df['sell_qty']\n    df['ask_buy_interaction'] = df['ask_qty'] * df['buy_qty']\n    df['ask_sell_interaction'] = df['ask_qty'] * df['sell_qty']\n\n\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'])\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'])\n    df['log_volume'] = np.log1p(df['volume'])\n\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'])\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'])\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'])\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'])\n    \n\n    return df\n\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n    \ndef load_data():\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n\n    train_df = add_features(train_df)\n    test_df = add_features(test_df)\n\n    Config.FEATURES += [\"log_volume\", 'bid_ask_interaction', 'bid_buy_interaction', 'bid_sell_interaction', 'ask_buy_interaction',\n                        'ask_sell_interaction']\n\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\n\ndef get_model_slices(n_samples: int):\n    return [\n        {\"name\": \"full_data\", \"cutoff\": 0},\n        {\"name\": \"last_90pct\", \"cutoff\": int(0.10 * n_samples)},\n        {\"name\": \"last_85pct\", \"cutoff\": int(0.15 * n_samples)},\n        {\"name\": \"last_80pct\", \"cutoff\": int(0.20 * n_samples)},\n\n    ]\n\n\n# =========================\n# Training and Evaluation\n# =========================\ndef train_and_evaluate(train_df, test_df):\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n\n    # Initialize prediction dictionaries for XGBoost\n    oof_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n        for learner in LEARNERS\n    }\n    test_preds = {\n        learner[\"name\"]: {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n        for learner in LEARNERS\n    }\n    \n    # Initialize prediction dictionaries for Neural Network\n    nn_oof_preds = {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n    nn_test_preds = {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n    nn_models = {s[\"name\"]: [] for s in model_slices}\n    nn_scalers = {s[\"name\"]: [] for s in model_slices}\n\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n\n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            subset = train_df.iloc[cutoff:].reset_index(drop=True)\n            rel_idx = train_idx[train_idx >= cutoff] - cutoff\n\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n\n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n\n            # Train XGBoost (unchanged)\n            for learner in LEARNERS:\n                print(f\"    Training XGBoost...\")\n                model = learner[\"Estimator\"](**learner[\"params\"])\n                model.fit(X_train, y_train, sample_weight=sw, eval_set=[(X_valid, y_valid)], verbose=False)\n\n                # XGBoost OOF predictions\n                mask = valid_idx >= cutoff\n                if mask.any():\n                    idxs = valid_idx[mask]\n                    oof_preds[learner[\"name\"]][slice_name][idxs] = model.predict(train_df.iloc[idxs][Config.FEATURES])\n                if cutoff > 0 and (~mask).any():\n                    oof_preds[learner[\"name\"]][slice_name][valid_idx[~mask]] = oof_preds[learner[\"name\"]][\"full_data\"][\n                        valid_idx[~mask]]\n\n                # XGBoost test predictions\n                test_preds[learner[\"name\"]][slice_name] += model.predict(test_df[Config.FEATURES])\n                \n                # Get XGBoost predictions for neural network training\n                xgb_train_pred = model.predict(X_train)\n                xgb_valid_pred = model.predict(X_valid)\n                \n                # Train Neural Network Meta-Learner\n                print(f\"    Training Neural Network Meta-Learner...\")\n                try:\n                    nn_model, nn_scaler, nn_train_pred, nn_valid_pred = train_meta_learner(\n                        X_train.values, y_train.values, xgb_train_pred,\n                        X_valid.values, y_valid.values, xgb_valid_pred,\n                        sample_weights=sw\n                    )\n                    \n                    # Store neural network components\n                    nn_models[slice_name].append(nn_model)\n                    nn_scalers[slice_name].append(nn_scaler)\n                    \n                    # Neural Network OOF predictions\n                    # nn_valid_pred corresponds to ALL validation samples (same order as X_valid)\n                    mask = valid_idx >= cutoff\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        # Use corresponding predictions for the masked validation indices\n                        nn_oof_preds[slice_name][idxs] = nn_valid_pred[mask]\n                    if cutoff > 0 and (~mask).any():\n                        nn_oof_preds[slice_name][valid_idx[~mask]] = nn_oof_preds[\"full_data\"][valid_idx[~mask]]\n                    \n                    # Neural Network test predictions\n                    # Prepare test features with XGBoost predictions\n                    xgb_test_pred = model.predict(test_df[Config.FEATURES])\n                    test_meta_features = np.column_stack([test_df[Config.FEATURES].values, xgb_test_pred])\n                    test_scaled = nn_scaler.transform(test_meta_features)\n                    \n                    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n                    nn_model.eval()\n                    with torch.no_grad():\n                        test_tensor = torch.FloatTensor(test_scaled).to(device)\n                        nn_test_pred = nn_model(test_tensor).cpu().numpy()\n                    \n                    nn_test_preds[slice_name] += nn_test_pred\n                    \n                    print(f\"      Neural Network trained successfully\")\n                    \n                except Exception as e:\n                    print(f\"      Neural Network training failed: {str(e)}\")\n                    # Fallback: use XGBoost predictions\n                    mask = valid_idx >= cutoff\n                    if mask.any():\n                        idxs = valid_idx[mask]\n                        nn_oof_preds[slice_name][idxs] = oof_preds[learner[\"name\"]][slice_name][idxs]\n                    if cutoff > 0 and (~mask).any():\n                        nn_oof_preds[slice_name][valid_idx[~mask]] = nn_oof_preds[\"full_data\"][valid_idx[~mask]]\n                    \n                    nn_test_preds[slice_name] += test_preds[learner[\"name\"]][slice_name]\n\n    # Normalize test predictions\n    for learner_name in test_preds:\n        for slice_name in test_preds[learner_name]:\n            test_preds[learner_name][slice_name] /= (Config.N_FOLDS-1)\n    \n    for slice_name in nn_test_preds:\n        nn_test_preds[slice_name] /= (Config.N_FOLDS-1)\n\n    return oof_preds, test_preds, model_slices, nn_oof_preds, nn_test_preds\n\n\n# =========================\n# Ensemble & Submission\n# =========================\ndef safe_ensemble_strategy(train_df, xgb_oof, xgb_test, nn_oof, nn_test):\n    \"\"\"\n    Safe ensemble strategy with fallback mechanism\n    Only uses neural network if it improves performance\n    \"\"\"\n    y_true = train_df[Config.LABEL_COLUMN]\n    \n    # Calculate individual scores\n    xgb_score = pearsonr(y_true, xgb_oof)[0]\n    nn_score = pearsonr(y_true, nn_oof)[0]\n    \n    print(f\"\\nModel Performance Comparison:\")\n    print(f\"XGBoost OOF Score: {xgb_score:.4f}\")\n    print(f\"Neural Network OOF Score: {nn_score:.4f}\")\n    \n    # Safety threshold - only ensemble if NN is competitive\n    improvement_threshold = -0.005  # Allow 0.5% degradation\n    \n    if nn_score < xgb_score + improvement_threshold:\n        print(f\"Neural Network performance is significantly worse. Using XGBoost only.\")\n        return \"xgb_only\", xgb_score, xgb_test\n    \n    # Try different ensemble weights\n    best_score = xgb_score\n    best_weight = 0.0\n    best_test = xgb_test\n    \n    print(f\"\\nTesting ensemble weights:\")\n    for weight in np.arange(0.1, 0.8, 0.1):\n        ensemble_oof = (1 - weight) * xgb_oof + weight * nn_oof\n        ensemble_score = pearsonr(y_true, ensemble_oof)[0]\n        print(f\"  Weight {weight:.1f}: {ensemble_score:.4f}\")\n        \n        if ensemble_score > best_score:\n            best_score = ensemble_score\n            best_weight = weight\n            best_test = (1 - weight) * xgb_test + weight * nn_test\n    \n    if best_weight == 0.0:\n        print(f\"No ensemble weight improves performance. Using XGBoost only.\")\n        return \"xgb_only\", xgb_score, xgb_test\n    else:\n        improvement = ((best_score - xgb_score) / xgb_score) * 100\n        print(f\"Best ensemble weight: {best_weight:.1f}\")\n        print(f\"Ensemble improvement: {improvement:+.2f}%\")\n        return \"ensemble\", best_score, best_test\n\ndef ensemble_and_submit(train_df, oof_preds, test_preds, submission_df, nn_oof_preds=None, nn_test_preds=None):\n    learner_name = 'xgb'\n    weights = np.array([1,1,1,1])\n\n    # XGBoost ensemble (original logic)\n    xgb_oof_weighted = pd.DataFrame(oof_preds[learner_name]).values @ weights\n    xgb_test_weighted = pd.DataFrame(test_preds[learner_name]).values @ weights\n    xgb_score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], xgb_oof_weighted)[0]\n    print(f\"{learner_name.upper()} Weighted Ensemble Pearson: {xgb_score_weighted:.4f}\")\n\n    # If neural network predictions are available, try ensemble\n    if nn_oof_preds is not None and nn_test_preds is not None:\n        print(f\"\\n\" + \"=\"*50)\n        print(\"NEURAL NETWORK ENSEMBLE EVALUATION\")\n        print(\"=\"*50)\n        \n        # Neural Network ensemble\n        nn_oof_weighted = pd.DataFrame(nn_oof_preds).values @ weights\n        nn_test_weighted = pd.DataFrame(nn_test_preds).values @ weights\n        nn_score_weighted = pearsonr(train_df[Config.LABEL_COLUMN], nn_oof_weighted)[0]\n        print(f\"Neural Network Weighted Ensemble Pearson: {nn_score_weighted:.4f}\")\n        \n        # Safe ensemble strategy\n        strategy, final_score, final_test = safe_ensemble_strategy(\n            train_df, xgb_oof_weighted, xgb_test_weighted, \n            nn_oof_weighted, nn_test_weighted\n        )\n        \n        print(f\"\\n\" + \"=\"*50)\n        print(\"FINAL MODEL SELECTION\")\n        print(\"=\"*50)\n        print(f\"Selected Strategy: {strategy}\")\n        print(f\"Final Score: {final_score:.4f}\")\n        \n        submission_df[\"prediction\"] = final_test\n        filename = f\"submission_with_nn_{strategy}.csv\"\n        \n    else:\n        print(\"Neural Network predictions not available. Using XGBoost only.\")\n        submission_df[\"prediction\"] = xgb_test_weighted\n        filename = \"submission_xgb_only.csv\"\n\n    submission_df.to_csv(filename, index=False)\n    print(f\"Saved: {filename}\")\n    print(submission_df.head(10))\n\nif __name__ == \"__main__\":\n    print(\"=\"*60)\n    print(\"XGBoost + Neural Network Meta-Learning Pipeline\")\n    print(\"=\"*60)\n    print(\"Strategy: Conservative ensemble with fallback mechanism\")\n    \n    train_df, test_df, submission_df = load_data()\n    oof_preds, test_preds, model_slices, nn_oof_preds, nn_test_preds = train_and_evaluate(train_df, test_df)\n    ensemble_and_submit(train_df, oof_preds, test_preds, submission_df, nn_oof_preds, nn_test_preds)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}