{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import KFold\nfrom xgboost import XGBRegressor\nfrom scipy.stats import pearsonr\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ===== Feature Engineering =====\ndef feature_engineering(df):\n    \"\"\"Original features plus new robust features\"\"\"\n    # Original features\n    df['volume_weighted_sell'] = df['sell_qty'] * df['volume']\n    df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-8)\n    df['selling_pressure'] = df['sell_qty'] / (df['volume'] + 1e-8)\n    df['effective_spread_proxy'] = np.abs(df['buy_qty'] - df['sell_qty']) / (df['volume'] + 1e-8)\n    # New robust features\n    df['log_volume'] = np.log1p(df['volume'])\n    df['bid_ask_imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-8)\n    df['order_flow_imbalance'] = (df['buy_qty'] - df['sell_qty']) / (df['buy_qty'] + df['sell_qty'] + 1e-8)\n    df['liquidity_ratio'] = (df['bid_qty'] + df['ask_qty']) / (df['volume'] + 1e-8)\n    # Handle infinities and NaN\n    df = df.replace([np.inf, -np.inf], np.nan)\n    for col in df.columns:\n        if df[col].isna().any():\n            median_val = df[col].median()\n            df[col] = df[col].fillna(median_val if not pd.isna(median_val) else 0)\n    return df\n\n# ===== Configuration =====\nclass Config:\n    TRAIN_PATH = \"/kaggle/input/drw-crypto-market-prediction/train.parquet\"\n    TEST_PATH = \"/kaggle/input/drw-crypto-market-prediction/test.parquet\"\n    SUBMISSION_PATH = \"/kaggle/input/drw-crypto-market-prediction/sample_submission.csv\"\n    FEATURES = [\n        \"X863\", \"X856\", \"X598\", \"X862\", \"X385\", \"X852\", \"X603\", \"X860\", \"X674\",\n        \"X415\", \"X345\", \"X855\", \"X174\", \"X302\", \"X178\", \"X168\", \"X612\",\n        \"buy_qty\", \"sell_qty\", \"volume\", \"X888\", \"X421\", \"X333\",\n        \"bid_qty\", \"ask_qty\"\n    ]\n    LABEL_COLUMN = \"label\"\n    N_FOLDS = 3\n    RANDOM_STATE = 42\n\nXGB_PARAMS = {\n    \"tree_method\": \"hist\",\n    \"device\": \"gpu\",  # Will fallback to CPU if GPU not available\n    \"colsample_bylevel\": 0.4778,\n    \"colsample_bynode\": 0.3628,\n    \"colsample_bytree\": 0.7107,\n    \"gamma\": 1.7095,\n    \"learning_rate\": 0.02213,\n    \"max_depth\": 20,\n    \"max_leaves\": 12,\n    \"min_child_weight\": 16,\n    \"n_estimators\": 1667,\n    \"subsample\": 0.06567,\n    \"reg_alpha\": 39.3524,\n    \"reg_lambda\": 75.4484,\n    \"verbosity\": 0,\n    \"random_state\": Config.RANDOM_STATE,\n    \"n_jobs\": -1\n}\n\n# ===== Data Loading & EDA =====\ndef create_time_decay_weights(n: int, decay: float = 0.9) -> np.ndarray:\n    \"\"\"Create time decay weights for more recent data importance\"\"\"\n    positions = np.arange(n)\n    normalized = positions / (n - 1)\n    weights = decay ** (1.0 - normalized)\n    return weights * n / weights.sum()\n\ndef load_data():\n    \"\"\"Load and preprocess data\"\"\"\n    train_df = pd.read_parquet(Config.TRAIN_PATH, columns=Config.FEATURES + [Config.LABEL_COLUMN])\n    test_df = pd.read_parquet(Config.TEST_PATH, columns=Config.FEATURES)\n    submission_df = pd.read_csv(Config.SUBMISSION_PATH)\n    # Apply feature engineering\n    train_df = feature_engineering(train_df)\n    test_df = feature_engineering(test_df)\n    engineered_features = [\n        \"volume_weighted_sell\", \"buy_sell_ratio\", \"selling_pressure\", \n        \"effective_spread_proxy\", \"log_volume\", \"bid_ask_imbalance\",\n        \"order_flow_imbalance\", \"liquidity_ratio\"\n    ]\n    Config.FEATURES = list(set(Config.FEATURES + engineered_features))\n    print(f\"Loaded data - Train: {train_df.shape}, Test: {test_df.shape}, Submission: {submission_df.shape}\")\n    print(f\"Total features: {len(Config.FEATURES)}\")\n    return train_df.reset_index(drop=True), test_df.reset_index(drop=True), submission_df\n\ndef perform_eda(df, title=\"Train Data\"):\n    \"\"\"Perform comprehensive exploratory data analysis\"\"\"\n    print(f\"\\n== {title} EDA ==\")\n    # 1. Basic Info\n    print(\"\\n--- Basic Info ---\")\n    print(df.info())\n    # 2. Missing Values\n    print(\"\\n--- Missing Values ---\")\n    print(df.isna().sum())\n    sns.heatmap(df.isna(), cbar=False, cmap='viridis')\n    plt.title(f\"{title} - Missing Values\")\n    plt.show()\n    # 3. Summary Statistics\n    print(\"\\n--- Summary Statistics ---\")\n    print(df.describe(include='all'))\n    # 4. Label Distribution\n    if Config.LABEL_COLUMN in df:\n        print(\"\\n--- Label Distribution ---\")\n        sns.histplot(df[Config.LABEL_COLUMN], kde=True)\n        plt.title(f\"{title} - Label Distribution\")\n        plt.show()\n    # 5. Numeric Feature Distributions (sample)\n    print(\"\\n--- Numeric Feature Distributions (Sample) ---\")\n    sample_features = [\"buy_qty\", \"sell_qty\", \"volume\", \"bid_qty\", \"ask_qty\"]\n    for feat in sample_features:\n        if feat in df:\n            sns.histplot(df[feat], kde=True)\n            plt.title(f\"{title} - {feat} Distribution\")\n            plt.show()\n    # 6. Correlation Matrix (for numeric features)\n    print(\"\\n--- Correlation Matrix (Sample Numeric Features) ---\")\n    numeric_cols = df.select_dtypes(include=np.number).columns\n    corr = df[numeric_cols].corr()\n    plt.figure(figsize=(12,8))\n    sns.heatmap(corr, annot=False, cmap='coolwarm', center=0)\n    plt.title(f\"{title} - Correlation Matrix\")\n    plt.show()\n    # 7. Time Series Plot (if index is time)\n    if not df.index.name and not isinstance(df.index, pd.MultiIndex):\n        if len(df) > 1000:\n            sample_size = 1000\n            sample_idx = np.linspace(0, len(df)-1, sample_size, dtype=int)\n            df_sample = df.iloc[sample_idx]\n        else:\n            df_sample = df.copy()\n        for feat in sample_features:\n            if feat in df_sample:\n                plt.plot(df_sample[feat], label=feat)\n        plt.legend()\n        plt.title(f\"{title} - Time Series (Sampled)\")\n        plt.show()\n\n# ===== Model Training =====\ndef get_model_slices(n_samples: int):\n    \"\"\"Define different data slices for training\"\"\"\n    return [\n        {\"name\": \"full_data\", \"cutoff\": 0},\n        {\"name\": \"last_90pct\", \"cutoff\": int(0.10 * n_samples)},\n        {\"name\": \"last_80pct\", \"cutoff\": int(0.20 * n_samples)},\n        {\"name\": \"last_75pct\", \"cutoff\": int(0.25 * n_samples)},\n        {\"name\": \"last_70pct\", \"cutoff\": int(0.30 * n_samples)},\n        {\"name\": \"last_60pct\", \"cutoff\": int(0.40 * n_samples)},\n        {\"name\": \"last_50pct\", \"cutoff\": int(0.50 * n_samples)},\n    ]\n\ndef train_xgboost_model(X_train, y_train, X_valid, y_valid, X_test, sample_weights=None):\n    \"\"\"Train XGBoost model\"\"\"\n    try:\n        model = XGBRegressor(**XGB_PARAMS)\n        model.fit(X_train, y_train, sample_weight=sample_weights, eval_set=[(X_valid, y_valid)], verbose=False)\n        valid_pred = model.predict(X_valid)\n        test_pred = model.predict(X_test)\n        return valid_pred, test_pred, model\n    except Exception as e:\n        print(f\"    Error training XGBoost: {str(e)}\")\n        return None, None, None\n\ndef train_and_evaluate(train_df, test_df):\n    \"\"\"Train XGBoost models with cross-validation on different data slices\"\"\"\n    n_samples = len(train_df)\n    model_slices = get_model_slices(n_samples)\n    oof_preds = {s[\"name\"]: np.zeros(n_samples) for s in model_slices}\n    test_preds = {s[\"name\"]: np.zeros(len(test_df)) for s in model_slices}\n    trained_models = {s[\"name\"]: [] for s in model_slices}\n    full_weights = create_time_decay_weights(n_samples)\n    kf = KFold(n_splits=Config.N_FOLDS, shuffle=False)\n    \n    for fold, (train_idx, valid_idx) in enumerate(kf.split(train_df), start=1):\n        print(f\"\\n--- Fold {fold}/{Config.N_FOLDS} ---\")\n        X_valid = train_df.iloc[valid_idx][Config.FEATURES]\n        y_valid = train_df.iloc[valid_idx][Config.LABEL_COLUMN]\n        X_test = test_df[Config.FEATURES]\n        \n        for s in model_slices:\n            cutoff = s[\"cutoff\"]\n            slice_name = s[\"name\"]\n            subset = train_df.iloc[cutoff:].reset_index(drop=True)\n            rel_idx = train_idx[train_idx >= cutoff] - cutoff\n            if len(rel_idx) == 0:\n                continue\n            X_train = subset.iloc[rel_idx][Config.FEATURES]\n            y_train = subset.iloc[rel_idx][Config.LABEL_COLUMN]\n            sw = create_time_decay_weights(len(subset))[rel_idx] if cutoff > 0 else full_weights[train_idx]\n            print(f\"  Training slice: {slice_name}, samples: {len(X_train)}\")\n            valid_pred, test_pred, model = train_xgboost_model(X_train, y_train, X_valid, y_valid, X_test, sw)\n            if valid_pred is None:\n                continue\n            mask = valid_idx >= cutoff\n            if mask.any():\n                oof_preds[slice_name][valid_idx[mask] if valid_idx[mask].size > 0 else valid_idx] = valid_pred[mask] if valid_pred[mask].size > 0 else valid_pred\n            if cutoff > 0 and (~mask).any():\n                oof_preds[slice_name][valid_idx[~mask]] = oof_preds[\"full_data\"][valid_idx[~mask]]\n            test_preds[slice_name] += test_pred\n            trained_models[slice_name].append(model)\n    \n    for slice_name in test_preds:\n        if len(trained_models[slice_name]) > 0:\n            test_preds[slice_name] /= len(trained_models[slice_name])\n    return oof_preds, test_preds, model_slices, trained_models\n\n# ===== Evaluation and Submission =====\ndef evaluate_and_create_submissions(train_df, oof_preds, test_preds, model_slices, submission_df):\n    \"\"\"Evaluate different strategies and create submissions\"\"\"\n    print(\"\\n\" + \"=\"*60)\n    print(\"EVALUATION RESULTS:\")\n    print(\"=\"*60)\n    slice_scores = {}\n    for s in model_slices:\n        slice_name = s[\"name\"]\n        if slice_name in oof_preds:\n            score = pearsonr(train_df[Config.LABEL_COLUMN], oof_preds[slice_name])[0]\n            slice_scores[slice_name] = score\n            print(f\"{slice_name:15s}: {score:.4f}\")\n    best_slice = max(slice_scores.items(), key=lambda x: x[1])\n    print(f\"\\nBest Strategy: {best_slice[0]} (Score: {best_slice[1]:.4f})\")\n    submissions_created = []\n    for s in model_slices:\n        slice_name = s[\"name\"]\n        if slice_name in test_preds:\n            submission = submission_df.copy()\n            submission[\"prediction\"] = test_preds[slice_name]\n            filename = f\"submission_xgb_{slice_name}.csv\"\n            submission.to_csv(filename, index=False)\n            submissions_created.append(filename)\n            print(f\"Created: {filename}\")\n    if len(slice_scores) > 1:\n        total_score = sum(max(0, score) for score in slice_scores.values())\n        if total_score > 0:\n            weighted_oof = np.zeros(len(train_df))\n            weighted_test = np.zeros(len(test_df))\n            for slice_name, score in slice_scores.items():\n                if score > 0:\n                    weight = score / total_score\n                    weighted_oof += weight * oof_preds[slice_name]\n                    weighted_test += weight * test_preds[slice_name]\n            ensemble_score = pearsonr(train_df[Config.LABEL_COLUMN], weighted_oof)[0]\n            print(f\"\\nWeighted Ensemble Score: {ensemble_score:.4f}\")\n            print(\"Weights:\")\n            for slice_name, score in slice_scores.items():\n                if score > 0:\n                    weight = score / total_score\n                    print(f\"  {slice_name}: {weight:.3f}\")\n            submission_ensemble = submission_df.copy()\n            submission_ensemble[\"prediction\"] = weighted_test\n            submission_ensemble.to_csv(\"submission_xgb_weighted_ensemble.csv\", index=False)\n            submissions_created.append(\"submission_xgb_weighted_ensemble.csv\")\n            print(\"Created: submission_xgb_weighted_ensemble.csv\")\n    submission_best = submission_df.copy()\n    submission_best[\"prediction\"] = test_preds[best_slice[0]]\n    submission_best.to_csv(\"submission_xgb_best.csv\", index=False)\n    submissions_created.append(\"submission_xgb_best.csv\")\n    print(\"Created: submission_xgb_best.csv (recommended)\")\n    print(\"\\n\" + \"=\"*60)\n    print(\"SUBMISSION SUMMARY:\")\n    print(\"=\"*60)\n    for slice_name, score in sorted(slice_scores.items(), key=lambda x: x[1], reverse=True):\n        print(f\"{slice_name:20s}: {score:.4f}\")\n    return submissions_created, slice_scores\n\n# ===== Main Execution =====\nif __name__ == \"__main__\":\n    print(\"🚀 XGBoost Crypto Prediction Pipeline\")\n    print(\"=\"*50)\n    print(\"Loading data...\")\n    train_df, test_df, submission_df = load_data()\n    print(\"\\nPerforming Exploratory Data Analysis...\")\n    perform_eda(train_df, \"Train Data\")\n    print(\"\\nTraining XGBoost models...\")\n    oof_preds, test_preds, model_slices, trained_models = train_and_evaluate(train_df, test_df)\n    print(\"\\nEvaluating and creating submissions...\")\n    submissions_created, slice_scores = evaluate_and_create_submissions(\n        train_df, oof_preds, test_preds, model_slices, submission_df\n    )\n    print(f\"\\n✅ Pipeline completed successfully!\")\n    print(f\"📁 Created {len(submissions_created)} submission files:\")\n    for filename in submissions_created:\n        print(f\"   - {filename}\")\n    print(f\"\\n🏆 Recommended submission: submission_xgb_best.csv\")\n    print(f\"📊 Best score: {max(slice_scores.values()):.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-26T06:45:41.626502Z","iopub.execute_input":"2025-06-26T06:45:41.626951Z","iopub.status.idle":"2025-06-26T07:23:47.281693Z","shell.execute_reply.started":"2025-06-26T06:45:41.626924Z","shell.execute_reply":"2025-06-26T07:23:47.280481Z"}},"outputs":[],"execution_count":null}]}