{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom datetime import datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Machine Learning Libraries\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import pearsonr\n\n# Gradient Boosting Models\nimport xgboost as xgb\nimport lightgbm as lgb\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\n\n# Set style and random seed\nsns.set_style('whitegrid')\nnp.random.seed(42)\n\n\nclass HybridCryptoPredictor:\n    \"\"\"\n    Hybrid prediction system that combines autoregressive features from labels\n    during training with market microstructure features for robust predictions.\n    \"\"\"\n    \n    def __init__(self, lookback_windows=None):\n        \"\"\"\n        Initialize the hybrid predictor.\n        \n        Parameters:\n        -----------\n        lookback_windows : list of int\n            Window sizes for creating lagged features\n        \"\"\"\n        self.lookback_windows = lookback_windows or [1, 5, 10, 20, 30, 60]\n        self.models = {}\n        self.scaler = RobustScaler()\n        self.feature_names = []\n        self.validation_scores = []\n        \n    def create_basic_features(self, df):\n        \"\"\"\n        Create basic features from market data that are available in both train and test.\n        These features ensure we can make predictions on the test set.\n        \"\"\"\n        features = pd.DataFrame(index=df.index)\n        \n        # Basic market features - these are available in both train and test\n        market_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n        \n        # Copy original features\n        for col in market_cols:\n            if col in df.columns:\n                features[col] = df[col].fillna(0)\n        \n        # Order book imbalance\n        if 'bid_qty' in df.columns and 'ask_qty' in df.columns:\n            features['order_imbalance'] = (\n                (df['bid_qty'] - df['ask_qty']) / \n                (df['bid_qty'] + df['ask_qty'] + 1e-8)\n            )\n        \n        # Trade flow imbalance\n        if 'buy_qty' in df.columns and 'sell_qty' in df.columns:\n            features['trade_imbalance'] = (\n                (df['buy_qty'] - df['sell_qty']) / \n                (df['buy_qty'] + df['sell_qty'] + 1e-8)\n            )\n        \n        # Volume ratios\n        if 'volume' in df.columns:\n            for col in ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty']:\n                if col in df.columns:\n                    features[f'{col}_volume_ratio'] = df[col] / (df['volume'] + 1e-8)\n        \n        # Log transforms for volume features\n        for col in market_cols:\n            if col in features.columns:\n                features[f'log_{col}'] = np.log1p(features[col])\n        \n        # Replace any infinities or NaN values\n        features = features.replace([np.inf, -np.inf], 0).fillna(0)\n        \n        return features\n    \n    def create_autoregressive_features(self, df, target_col='label'):\n        \"\"\"\n        Create autoregressive features from the target variable.\n        Only used during training when we have access to historical labels.\n        \"\"\"\n        ar_features = pd.DataFrame(index=df.index)\n        \n        if target_col in df.columns:\n            target = df[target_col]\n            \n            # Create lagged features\n            for window in self.lookback_windows:\n                ar_features[f'target_lag_{window}'] = target.shift(window)\n                \n                # Rolling statistics\n                ar_features[f'target_ma_{window}'] = target.rolling(\n                    window=window, min_periods=1\n                ).mean()\n                \n                ar_features[f'target_std_{window}'] = target.rolling(\n                    window=window, min_periods=1\n                ).std()\n                \n                # Momentum features\n                ar_features[f'target_momentum_{window}'] = target - target.shift(window)\n        \n        # Fill NaN values\n        ar_features = ar_features.fillna(0)\n        \n        return ar_features\n    \n    def prepare_features(self, df, is_training=True):\n        \"\"\"\n        Prepare all features for modeling.\n        \"\"\"\n        # Get basic market features (available for both train and test)\n        basic_features = self.create_basic_features(df)\n        \n        if is_training:\n            # Add autoregressive features during training\n            ar_features = self.create_autoregressive_features(df)\n            features = pd.concat([basic_features, ar_features], axis=1)\n        else:\n            # Test set only has basic features\n            features = basic_features\n        \n        # Add a subset of the anonymized features (X_1 to X_50 to keep it simple)\n        anon_cols = [col for col in df.columns if col.startswith('X_')][:50]\n        for col in anon_cols:\n            if col in df.columns:\n                features[col] = df[col].fillna(0).replace([np.inf, -np.inf], 0)\n        \n        self.feature_names = features.columns.tolist()\n        return features\n    \n    def train_model(self, X_train, y_train, X_val=None, y_val=None):\n        \"\"\"\n        Train an ensemble of models.\n        \"\"\"\n        print(\"Training ensemble models...\")\n        \n        # Scale features\n        X_train_scaled = self.scaler.fit_transform(X_train)\n        \n        # XGBoost model\n        self.models['xgb'] = xgb.XGBRegressor(\n            objective='reg:squarederror',\n            n_estimators=800,\n            learning_rate=0.03,\n            max_depth=5,\n            min_child_weight=3,\n            subsample=0.8,\n            colsample_bytree=0.7,\n            gamma=0.05,\n            reg_alpha=0.05,\n            reg_lambda=1.0,\n            random_state=42,\n            n_jobs=-1\n        )\n        \n        # LightGBM model\n        self.models['lgb'] = lgb.LGBMRegressor(\n            objective='regression',\n            n_estimators=800,\n            learning_rate=0.03,\n            num_leaves=31,\n            feature_fraction=0.7,\n            bagging_fraction=0.8,\n            bagging_freq=5,\n            lambda_l1=0.05,\n            lambda_l2=1.0,\n            min_data_in_leaf=20,\n            random_state=42,\n            n_jobs=-1\n        )\n        \n        # Train models\n        if X_val is not None and y_val is not None:\n            X_val_scaled = self.scaler.transform(X_val)\n            \n            # Train XGBoost\n            self.models['xgb'].fit(\n                X_train_scaled, y_train,\n                eval_set=[(X_val_scaled, y_val)],\n                early_stopping_rounds=50,\n                verbose=False\n            )\n            \n            # Train LightGBM\n            self.models['lgb'].fit(\n                X_train_scaled, y_train,\n                eval_set=[(X_val_scaled, y_val)],\n                callbacks=[lgb.early_stopping(50), lgb.log_evaluation(0)]\n            )\n            \n            # Calculate validation scores\n            val_scores = {}\n            for name, model in self.models.items():\n                val_pred = model.predict(X_val_scaled)\n                val_corr, _ = pearsonr(y_val, val_pred)\n                val_scores[name] = val_corr\n                print(f\"{name} validation correlation: {val_corr:.4f}\")\n            \n            # Ensemble prediction\n            ensemble_pred = np.mean([\n                self.models['xgb'].predict(X_val_scaled),\n                self.models['lgb'].predict(X_val_scaled)\n            ], axis=0)\n            ensemble_corr, _ = pearsonr(y_val, ensemble_pred)\n            print(f\"Ensemble validation correlation: {ensemble_corr:.4f}\")\n            \n            return ensemble_corr\n        else:\n            # Train without validation\n            self.models['xgb'].fit(X_train_scaled, y_train)\n            self.models['lgb'].fit(X_train_scaled, y_train)\n            return None\n    \n    def predict(self, features):\n        \"\"\"\n        Generate ensemble predictions.\n        \"\"\"\n        features_scaled = self.scaler.transform(features)\n        \n        predictions = []\n        for name, model in self.models.items():\n            pred = model.predict(features_scaled)\n            predictions.append(pred)\n        \n        # Simple average ensemble\n        ensemble_pred = np.mean(predictions, axis=0)\n        return ensemble_pred\n    \n    def time_series_validation(self, features, labels, n_splits=5):\n        \"\"\"\n        Perform time series cross-validation.\n        \"\"\"\n        print(f\"\\nPerforming {n_splits}-fold time series cross-validation...\")\n        \n        tscv = TimeSeriesSplit(n_splits=n_splits)\n        cv_scores = []\n        \n        for fold, (train_idx, val_idx) in enumerate(tscv.split(features)):\n            print(f\"\\nFold {fold + 1}/{n_splits}\")\n            print(f\"Train size: {len(train_idx)}, Val size: {len(val_idx)}\")\n            \n            X_train_fold = features.iloc[train_idx]\n            y_train_fold = labels.iloc[train_idx]\n            X_val_fold = features.iloc[val_idx]\n            y_val_fold = labels.iloc[val_idx]\n            \n            fold_score = self.train_model(\n                X_train_fold, y_train_fold,\n                X_val_fold, y_val_fold\n            )\n            \n            if fold_score is not None:\n                cv_scores.append(fold_score)\n        \n        self.validation_scores = cv_scores\n        print(f\"\\nCV scores: {[f'{s:.4f}' for s in cv_scores]}\")\n        print(f\"Mean CV score: {np.mean(cv_scores):.4f} (+/- {np.std(cv_scores):.4f})\")\n        \n        return cv_scores\n\n\ndef main():\n    \"\"\"\n    Main execution pipeline.\n    \"\"\"\n    print(\"DRW Crypto Market Prediction - Hybrid Baseline Solution\")\n    print(\"=\" * 60)\n    \n    # Load training data\n    print(\"\\nLoading training data...\")\n    train_df = pd.read_parquet(\n        '/kaggle/input/drw-crypto-market-prediction/train.parquet',\n        engine='pyarrow'\n    )\n    print(f\"Training data shape: {train_df.shape}\")\n    \n    # Inspect columns\n    print(\"\\nInspecting data columns...\")\n    print(f\"Total columns: {len(train_df.columns)}\")\n    \n    # Check for timestamp column\n    if 'timestamp' in train_df.columns:\n        print(\"Timestamp column found\")\n        train_df['timestamp'] = pd.to_datetime(train_df['timestamp'])\n        cutoff_date = train_df['timestamp'].max() - pd.Timedelta(days=240)\n        train_df = train_df[train_df['timestamp'] >= cutoff_date].reset_index(drop=True)\n        print(f\"Using data from {cutoff_date} onwards ({len(train_df)} rows)\")\n    else:\n        print(\"No timestamp column found - using all data\")\n        # Use last portion of data assuming temporal ordering\n        cutoff_idx = max(0, len(train_df) - 300000)  # Use approximately last 300k rows\n        train_df = train_df.iloc[cutoff_idx:].reset_index(drop=True)\n        print(f\"Using last {len(train_df)} rows of data\")\n    \n    # Display available columns\n    market_cols = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', 'label']\n    available_market_cols = [col for col in market_cols if col in train_df.columns]\n    print(f\"\\nAvailable market columns: {available_market_cols}\")\n    \n    anonymized_cols = [col for col in train_df.columns if col.startswith('X_')]\n    print(f\"Number of anonymized features: {len(anonymized_cols)}\")\n    \n    # Basic statistics on label\n    if 'label' in train_df.columns:\n        print(f\"\\nLabel statistics:\")\n        print(f\"  Mean: {train_df['label'].mean():.6f}\")\n        print(f\"  Std: {train_df['label'].std():.6f}\")\n        print(f\"  Min: {train_df['label'].min():.6f}\")\n        print(f\"  Max: {train_df['label'].max():.6f}\")\n    \n    # Initialize predictor\n    predictor = HybridCryptoPredictor()\n    \n    # Prepare features\n    print(\"\\nPreparing features...\")\n    train_features = predictor.prepare_features(train_df, is_training=True)\n    train_labels = train_df['label']\n    \n    print(f\"\\nFeature matrix shape: {train_features.shape}\")\n    print(f\"Features: {len(train_features.columns)} total\")\n    print(f\"  - Market features: {len([c for c in train_features.columns if not c.startswith('X_') and not c.startswith('target_')])}\")\n    print(f\"  - Autoregressive features: {len([c for c in train_features.columns if c.startswith('target_')])}\")\n    print(f\"  - Anonymized features: {len([c for c in train_features.columns if c.startswith('X_')])}\")\n    \n    # Time series validation\n    cv_scores = predictor.time_series_validation(train_features, train_labels, n_splits=5)\n    \n    # Train final model\n    print(\"\\nTraining final model on all data...\")\n    predictor.train_model(train_features, train_labels)\n    \n    # Feature importance\n    print(\"\\nTop 20 most important features:\")\n    for name, model in predictor.models.items():\n        if hasattr(model, 'feature_importances_'):\n            importance_df = pd.DataFrame({\n                'feature': train_features.columns,\n                'importance': model.feature_importances_\n            }).sort_values('importance', ascending=False).head(20)\n            print(f\"\\n{name.upper()} top features:\")\n            for idx, row in importance_df.iterrows():\n                print(f\"  {row['feature']}: {row['importance']:.4f}\")\n    \n    # Generate test predictions\n    print(\"\\nLoading test data...\")\n    test_df = pd.read_parquet(\n        '/kaggle/input/drw-crypto-market-prediction/test.parquet',\n        engine='pyarrow'\n    )\n    print(f\"Test data shape: {test_df.shape}\")\n    \n    # Prepare test features (no autoregressive features available)\n    print(\"\\nPreparing test features...\")\n    test_features = predictor.prepare_features(test_df, is_training=False)\n    \n    # Ensure test features match training features\n    # Add missing columns with zeros\n    for col in train_features.columns:\n        if col not in test_features.columns:\n            test_features[col] = 0\n    \n    # Reorder columns to match training\n    test_features = test_features[train_features.columns]\n    \n    print(f\"Test feature matrix shape: {test_features.shape}\")\n    \n    # Generate predictions\n    print(\"\\nGenerating predictions...\")\n    predictions = predictor.predict(test_features)\n    \n    # Create submission\n    submission = pd.read_csv('/kaggle/input/drw-crypto-market-prediction/sample_submission.csv')\n    submission['prediction'] = predictions\n    submission.to_csv('submission.csv', index=False)\n    \n    print(\"\\nSubmission created successfully!\")\n    print(f\"\\nPrediction statistics:\")\n    print(f\"  Mean: {predictions.mean():.6f}\")\n    print(f\"  Std: {predictions.std():.6f}\")\n    print(f\"  Min: {predictions.min():.6f}\")\n    print(f\"  Max: {predictions.max():.6f}\")\n    \n    # Visualizations\n    fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n    \n    # Distribution of predictions\n    axes[0, 0].hist(predictions, bins=50, edgecolor='black', alpha=0.7)\n    axes[0, 0].set_xlabel('Predicted Values')\n    axes[0, 0].set_ylabel('Frequency')\n    axes[0, 0].set_title('Distribution of Test Predictions')\n    \n    # Sample predictions\n    axes[0, 1].plot(predictions[:500], alpha=0.7, linewidth=0.8)\n    axes[0, 1].set_xlabel('Sample Index')\n    axes[0, 1].set_ylabel('Predicted Value')\n    axes[0, 1].set_title('First 500 Test Predictions')\n    \n    # Training label distribution vs predictions\n    axes[1, 0].hist(train_labels, bins=50, alpha=0.5, label='Training Labels', density=True)\n    axes[1, 0].hist(predictions, bins=50, alpha=0.5, label='Test Predictions', density=True)\n    axes[1, 0].set_xlabel('Value')\n    axes[1, 0].set_ylabel('Density')\n    axes[1, 0].set_title('Distribution Comparison')\n    axes[1, 0].legend()\n    \n    # CV scores\n    if cv_scores:\n        axes[1, 1].plot(range(1, len(cv_scores) + 1), cv_scores, 'o-', markersize=8)\n        axes[1, 1].axhline(np.mean(cv_scores), color='red', linestyle='--', label=f'Mean: {np.mean(cv_scores):.4f}')\n        axes[1, 1].set_xlabel('Fold')\n        axes[1, 1].set_ylabel('Pearson Correlation')\n        axes[1, 1].set_title('Cross-Validation Performance')\n        axes[1, 1].legend()\n        axes[1, 1].grid(True, alpha=0.3)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return predictor, submission\n\n\nif __name__ == \"__main__\":\n    predictor, submission = main()","metadata":{"_uuid":"0181befa-e1bb-4f33-a0b7-48babdbcc6d3","_cell_guid":"0e151168-ae3c-4199-922e-686ad749bc6f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-05-23T04:33:44.570026Z","iopub.execute_input":"2025-05-23T04:33:44.570865Z","iopub.status.idle":"2025-05-23T04:36:57.136880Z","shell.execute_reply.started":"2025-05-23T04:33:44.570835Z","shell.execute_reply":"2025-05-23T04:36:57.135736Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"_uuid":"95fb4313-f842-4c8c-ba61-89fad43c2ea0","_cell_guid":"a180ee29-b5ea-4f54-9335-1103c820b942","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"","metadata":{"_uuid":"f33943dc-d253-4bd4-9472-1f89589de9ac","_cell_guid":"8a27004c-2e74-4373-896b-25e9316998db","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-05-22T14:50:26.404517Z","iopub.execute_input":"2025-05-22T14:50:26.404859Z","iopub.status.idle":"2025-05-22T14:51:02.948646Z","shell.execute_reply.started":"2025-05-22T14:50:26.404835Z","shell.execute_reply":"2025-05-22T14:51:02.947978Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"21d3f99c-4279-4f8b-966e-31298c64ddc0","_cell_guid":"7e2483ef-7b28-4992-836f-661b903d7f93","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-05-22T14:51:02.949538Z","iopub.execute_input":"2025-05-22T14:51:02.949751Z","iopub.status.idle":"2025-05-22T14:51:04.354521Z","shell.execute_reply.started":"2025-05-22T14:51:02.949732Z","shell.execute_reply":"2025-05-22T14:51:04.353584Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}