{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":96164,"databundleVersionId":12993472,"sourceType":"competition"},{"sourceId":249869065,"sourceType":"kernelVersion"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Step 1: Installations\n!pip install catboost lightgbm xgboost prophet -q\n\n# Step 2: Imports\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport traceback\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.feature_selection import SelectKBest, f_regression\nfrom sklearn.metrics import mean_absolute_error\nfrom scipy.stats import pearsonr\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\nfrom prophet import Prophet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3: Class Definition\nclass CryptoMarketPredictor:\n    \"\"\"\n    An optimized pipeline using an ensemble of tree-based models with rich,\n    two-pass feature engineering to predict crypto market movements.\n    \"\"\"\n    def __init__(self, top_features=100, top_X_features_to_preselect=30, use_future_lags=False):\n        self.top_features = top_features\n        self.top_X_features_to_preselect = top_X_features_to_preselect\n        self.scaler = RobustScaler()\n        self.feature_selector = None\n        self.selected_features = None\n        self.models = {}\n        self.preselected_X_n_names = None\n        self.use_future_lags = use_future_lags\n\n    def optimize_memory(self, df):\n        \"\"\"Optimize DataFrame memory usage.\"\"\"\n        print(f\"Memory usage before optimization: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n        for col in df.columns:\n            col_type = df[col].dtype\n            if col_type != 'object' and col not in ['timestamp', 'ID', 'label']:\n                c_min, c_max = df[col].min(), df[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max: df[col] = df[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max: df[col] = df[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max: df[col] = df[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max: df[col] = df[col].astype(np.int64)\n                else:\n                    if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max: df[col] = df[col].astype(np.float32)\n        print(f\"Memory usage after optimization: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n        return df\n\n    def clean_data(self, df):\n        \"\"\"Clean DataFrame by handling inf, NaN values.\"\"\"\n        df.replace([np.inf, -np.inf], np.nan, inplace=True)\n        for col in df.columns:\n            if df[col].dtype.kind in 'fc': # Check for float or complex types\n                df[col] = df[col].ffill().bfill()\n        df.fillna(0, inplace=True)\n        return df\n\n    def create_advanced_features(self, df, top_base_features):\n        \"\"\"\n        Engineer a rich set of time-based and interaction features.\n        This is the **ENHANCED** feature creation method.\n        \"\"\"\n        print(\"🛠️ Engineering rich time-based and interaction features...\")\n\n        # Basic price and imbalance features\n        if 'ask_qty' in df.columns and 'bid_qty' in df.columns:\n            df['mid_price'] = (df['ask_qty'] + df['bid_qty']) / 2\n            df['spread'] = df['ask_qty'] - df['bid_qty']\n            df['imbalance'] = (df['bid_qty'] - df['ask_qty']) / (df['bid_qty'] + df['ask_qty'] + 1e-10)\n            df['bid_ask_ratio'] = df['bid_qty'] / (df['ask_qty'] + 1e-10)\n        \n        if 'buy_qty' in df.columns and 'sell_qty' in df.columns:\n            df['buy_sell_ratio'] = df['buy_qty'] / (df['sell_qty'] + 1e-10)\n\n        # --- NEW: Rolling Statistics & Interactions based on Top Features ---\n        windows = [5, 10, 20, 30]\n        # Use only the top N features passed to this function for these complex calculations\n        features_for_adv_calcs = [f for f in top_base_features if f in df.columns]\n        \n        print(f\"  Creating rolling stats for top features: {features_for_adv_calcs[:5]}...\")\n        for feature in features_for_adv_calcs[:10]: # Limit to top 10 to manage feature explosion\n            for window in windows:\n                df[f'{feature}_ma_{window}'] = df[feature].rolling(window, min_periods=1).mean()\n                df[f'{feature}_vol_{window}'] = df[feature].rolling(window, min_periods=1).std()\n        \n        print(f\"  Creating interactions for top features: {features_for_adv_calcs[:5]}...\")\n        top_5_for_interactions = features_for_adv_calcs[:5]\n        for i in range(len(top_5_for_interactions)):\n            for j in range(i + 1, len(top_5_for_interactions)):\n                f1, f2 = top_5_for_interactions[i], top_5_for_interactions[j]\n                df[f'{f1}_{f2}_ratio'] = (df[f1] / (df[f2].abs() + 1e-10))\n                df[f'{f1}_{f2}_diff'] = (df[f1] - df[f2])\n\n        # Lag features\n        lag_periods = [1, 2, 5, 10]\n        cols_for_lags = [f for f in ['mid_price', 'imbalance'] if f in df.columns]\n        for col in cols_for_lags:\n            for lag in lag_periods:\n                if self.use_future_lags: df[f'{col}_lag_{lag}'] = df[col].shift(-lag)\n                else: df[f'{col}_lag_{lag}'] = df[col].shift(lag)\n\n        df = self.clean_data(df)\n        print(f\"Feature engineering complete. Shape: {df.shape}\")\n        return df\n\n    def select_features(self, X_df, y_df):\n        \"\"\"Select top k features using f_regression.\"\"\"\n        print(f\"Selecting top {self.top_features} features from {X_df.shape[1]}...\")\n        n_features = min(self.top_features, X_df.shape[1])\n        selector = SelectKBest(score_func=f_regression, k=n_features)\n        X_df_clean = X_df.replace([np.inf, -np.inf], 0).fillna(0)\n        y_df_clean = y_df.replace([np.inf, -np.inf], 0).fillna(0)\n        if y_df_clean.nunique() <= 1:\n            print(\"Warning: Target variable is constant. Skipping feature selection.\")\n            self.selected_features = X_df_clean.columns.tolist()\n            return X_df_clean.values\n        selector.fit(X_df_clean, y_df_clean)\n        self.feature_selector = selector\n        self.selected_features = X_df_clean.columns[selector.get_support()].tolist()\n        print(f\"Selected {len(self.selected_features)} features.\")\n        return X_df_clean[self.selected_features].values\n\n    def evaluate_model(self, y_true, y_pred, model_name):\n        \"\"\"Evaluate model performance.\"\"\"\n        mae = mean_absolute_error(y_true, y_pred)\n        correlation, _ = pearsonr(y_true, y_pred) if np.std(y_true) > 0 and np.std(y_pred) > 0 else (np.nan, np.nan)\n        print(f\"📊 {model_name} - MAE: {mae:.4f}, Pearson Correlation: {correlation:.4f}\")\n        return correlation\n\n    def train_lightgbm(self, X_train, y_train, X_val, y_val):\n        params = {'objective':'regression_l1','metric':'mae','n_estimators':1500,'learning_rate':0.03,'feature_fraction':0.8,'bagging_fraction':0.8,'bagging_freq':1,'lambda_l1':0.1,'lambda_l2':0.1,'num_leaves':31,'verbose':-1,'n_jobs':-1,'seed':42}\n        model = lgb.LGBMRegressor(**params)\n        model.fit(X_train, y_train, eval_set=[(X_val, y_val)], callbacks=[lgb.early_stopping(100, verbose=False)])\n        return model\n\n    def train_xgboost(self, X_train, y_train, X_val, y_val):\n        params = {'objective':'reg:squarederror','eval_metric':'mae','n_estimators':1500,'learning_rate':0.03,'tree_method':'hist','subsample':0.8,'colsample_bytree':0.8,'seed':42,'n_jobs':-1}\n        model = xgb.XGBRegressor(**params)\n        model.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=100, verbose=False)\n        return model\n\n    def train_catboost(self, X_train, y_train, X_val, y_val):\n        params = {'objective':'MAE','eval_metric':'MAE','iterations':1500,'learning_rate':0.03,'random_seed':42,'logging_level':'Silent','l2_leaf_reg':3,'bagging_temperature':1}\n        model = cb.CatBoostRegressor(**params)\n        model.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=100, verbose=False)\n        return model\n\n    def fit(self, train_data_raw):\n        \"\"\"Main training pipeline.\"\"\"\n        print(\"🚀 Starting training pipeline...\")\n        train_data_raw['timestamp'] = pd.to_datetime(train_data_raw['timestamp'])\n\n        print(\"Filtering data to the last 3 months for relevance...\")\n        three_months_prior = train_data_raw['timestamp'].max() - pd.DateOffset(months=3)\n        train_df = train_data_raw[train_data_raw['timestamp'] >= three_months_prior].copy()\n        train_df.sort_values(by='timestamp', inplace=True)\n        train_df.reset_index(drop=True, inplace=True)\n        del train_data_raw; gc.collect()\n\n        # --- Pass 1: Pre-select top 'X' features ---\n        print(\"\\n--- Pass 1: Pre-selecting top anonymous 'X' features ---\")\n        sample_df = train_df.sample(n=min(50000, len(train_df)), random_state=42)\n        X_n_cols_raw = [c for c in sample_df.columns if c.startswith('X')]\n        pre_selector = SelectKBest(score_func=f_regression, k=self.top_X_features_to_preselect)\n        pre_selector.fit(sample_df[X_n_cols_raw].fillna(0), sample_df['label'].fillna(0))\n        self.preselected_X_n_names = [col for col, support in zip(X_n_cols_raw, pre_selector.get_support()) if support]\n        print(f\"Pre-selected {len(self.preselected_X_n_names)} 'X' features.\")\n        del sample_df; gc.collect()\n\n        # --- Pass 2: Create Advanced Features ---\n        print(\"\\n--- Pass 2: Engineering advanced features on pre-selected data ---\")\n        base_cols = ['timestamp', 'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', 'label']\n        cols_to_use = list(dict.fromkeys(base_cols + self.preselected_X_n_names))\n        cols_to_use = [c for c in cols_to_use if c in train_df.columns]\n        train_df = train_df[cols_to_use]\n        train_df = self.optimize_memory(train_df)\n        # Pass the pre-selected 'X' names to the advanced feature creator\n        train_df = self.create_advanced_features(train_df, top_base_features=self.preselected_X_n_names)\n        \n        feature_cols = [c for c in train_df.columns if c not in ['timestamp', 'ID', 'label']]\n        X_df = train_df[feature_cols]\n        y_df = train_df['label']\n\n        # --- Final Feature Selection & Scaling ---\n        X_selected_array = self.select_features(X_df, y_df)\n        X_scaled = self.scaler.fit_transform(X_selected_array)\n        \n        # --- Time-Based Validation Split ---\n        val_size = int(len(X_scaled) * 0.2)\n        train_size = len(X_scaled) - val_size\n        X_train, X_val = X_scaled[:train_size], X_scaled[train_size:]\n        y_train, y_val = y_df.iloc[:train_size], y_df.iloc[train_size:]\n        print(f\"\\nTrain set size: {len(X_train)}, Validation set size: {len(X_val)}\")\n        del X_df, y_df, X_selected_array, X_scaled, train_df; gc.collect()\n\n        # --- Train Models ---\n        self.models['lgb'] = self.train_lightgbm(X_train, y_train, X_val, y_val)\n        self.models['xgb'] = self.train_xgboost(X_train, y_train, X_val, y_val)\n        self.models['cat'] = self.train_catboost(X_train, y_train, X_val, y_val)\n\n        # --- Evaluate Ensemble ---\n        print(\"\\n--- Validation Set Evaluation ---\")\n        val_preds = {name: model.predict(X_val) for name, model in self.models.items()}\n        for name, pred in val_preds.items(): self.evaluate_model(y_val, pred, name.upper())\n        self.evaluate_model(y_val, np.mean(list(val_preds.values()), axis=0), \"ENSEMBLE\")\n        print(\"\\n✅ Training pipeline complete.\")\n        return self\n\n    def predict(self, test_data_raw):\n        \"\"\"Generate predictions for test data in memory-managed chunks.\"\"\"\n        print(\"\\nGenerating predictions...\")\n        if not self.models: raise ValueError(\"Models must be trained first. Call fit().\")\n        \n        # This will be returned for final mapping\n        test_ids_in_prediction_order = test_data_raw['ID'].copy()\n\n        # Feature Engineering on the whole (sorted) test set\n        base_cols = ['timestamp', 'bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume']\n        cols_to_use = list(dict.fromkeys(base_cols + self.preselected_X_n_names))\n        cols_to_use = [c for c in cols_to_use if c in test_data_raw.columns]\n        X_test = test_data_raw[cols_to_use]\n        X_test = self.create_advanced_features(X_test, top_base_features=self.preselected_X_n_names)\n        \n        # Ensure all selected features are present\n        for f in self.selected_features:\n            if f not in X_test.columns: X_test[f] = 0\n        X_test = X_test[self.selected_features]\n\n        # Scaling\n        X_test_scaled = self.scaler.transform(X_test)\n        X_test_scaled = np.nan_to_num(X_test_scaled)\n        \n        # Prediction\n        preds = {name: model.predict(X_test_scaled) for name, model in self.models.items()}\n        final_predictions = np.mean(list(preds.values()), axis=0)\n\n        print(f\"Generated {len(final_predictions)} predictions.\")\n        return final_predictions, test_ids_in_prediction_order\n\n# --- Prophet Enhancement Function ---\ndef train_prophet_enhancement(df_ensemble, prophet_weight=0.085, sample_size=100000):\n    \"\"\"\n    Train a Prophet model on a SAMPLE and then PREDICT IN CHUNKS to save memory.\n    \"\"\"\n    print(\"\\n--- 🔮 Starting Prophet Enhancement ---\")\n    \n    # --- 1. FIT ON A SAMPLE (No change here) ---\n    if len(df_ensemble) > sample_size:\n        print(f\"Training Prophet on a sample of {sample_size} rows...\")\n        df_sample = df_ensemble.sample(n=sample_size, random_state=42)\n    else:\n        df_sample = df_ensemble\n\n    prophet_df_train = pd.DataFrame()\n    base_date = pd.to_datetime('2024-01-01')\n    prophet_df_train['ds'] = base_date + pd.to_timedelta(df_sample['ID'] - df_sample['ID'].min(), unit='H')\n    prophet_df_train['y'] = df_sample['Prediction'].values\n\n    model = Prophet(\n        growth='linear',\n        changepoint_prior_scale=0.06,\n        yearly_seasonality=False,  # Disabling this saves a lot of memory\n        weekly_seasonality=True,\n        daily_seasonality=False,\n        seasonality_mode='multiplicative',\n    )\n    \n    print(\"Fitting Prophet model on the sample...\")\n    model.fit(prophet_df_train)\n    \n    # --- 2. PREDICT IN CHUNKS (This is the fix) ---\n    print(\"Predicting for the full dataset in chunks to save memory...\")\n    future_df = pd.DataFrame()\n    future_df['ds'] = base_date + pd.to_timedelta(df_ensemble['ID'] - df_ensemble['ID'].min(), unit='H')\n    \n    chunk_size = 100000  # Process 100k rows at a time\n    all_forecasts = []\n    \n    for i in range(0, len(future_df), chunk_size):\n        chunk = future_df.iloc[i:i + chunk_size]\n        forecast_chunk = model.predict(chunk)\n        all_forecasts.append(forecast_chunk)\n        \n    forecast = pd.concat(all_forecasts, ignore_index=True)\n    prophet_predictions = forecast['yhat'].values\n    # --- END OF FIX ---\n    \n    # --- 3. BLEND RESULTS (No change here) ---\n    ensemble_predictions = df_ensemble['Prediction'].values\n    final_predictions = (1 - prophet_weight) * ensemble_predictions + prophet_weight * prophet_predictions\n    \n    df_final_prophet = pd.DataFrame({'ID': df_ensemble['ID'], 'Prediction': final_predictions})\n    \n    print(\"--- Prophet Enhancement Complete ---\")\n    return df_final_prophet","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Main Execution Function ---\ndef main():\n    \"\"\"A wrapper function to run the entire pipeline and manage memory.\"\"\"\n    try:\n        # 1. LOAD DATA\n        print(\"Loading data...\")\n        train_full_raw = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/train.parquet')\n        test_full_raw = pd.read_parquet('/kaggle/input/drw-crypto-market-prediction/test.parquet')\n\n        def setup_dataframe(df, name):\n            print(f\"  Setting up {name} DataFrame...\")\n            # The index is the timestamp, so reset it and rename it.\n            df.reset_index(inplace=True)\n            df.rename(columns={'index': 'timestamp'}, inplace=True)\n        \n            # --- FIX: Only create an 'ID' column if one doesn't already exist ---\n            if 'ID' not in df.columns:\n                print(\"  'ID' column not found. Creating one from the index.\")\n                df['ID'] = df.index\n            else:\n                print(\"  'ID' column already exists. Using original IDs.\")\n            # --- END FIX ---\n            \n            return df\n\n        train_full_raw = setup_dataframe(train_full_raw, \"train\")\n        test_full_raw = setup_dataframe(test_full_raw, \"test\")\n\n        original_shuffled_ids = test_full_raw['ID'].copy()\n\n        # 2. TIMESTAMP RECONSTRUCTION (The Critical Step 🤫)\n        print(\"\\nApplying timestamp reconstruction...\")\n        timestamp_recon_path = '/kaggle/input/the-order-of-the-test-rows-2/closest_rows.csv'\n        \n        # --- FIX PART 1: REMOVED THE TRY...EXCEPT BLOCK TO ENFORCE FAILURE ---\n        if os.path.exists(timestamp_recon_path):\n            print(\"  Loading reconstruction file...\")\n            t_recon = pd.read_csv(timestamp_recon_path, header=None).iloc[:, 0]\n\n            reorder_map = pd.DataFrame({\n                'original_pos': np.arange(len(t_recon)),\n                'chrono_pos': t_recon.to_numpy()\n            })\n\n            valid_matches = reorder_map[reorder_map['chrono_pos'] != -1].copy()\n            print(f\"  Found {len(valid_matches)} valid matches.\")\n\n            valid_matches.sort_values('chrono_pos', inplace=True)\n            sorted_indices = valid_matches['original_pos'].to_numpy()\n\n            # --- FIX PART 2: DEFENSIVE CHECK FOR OUT-OF-BOUNDS INDICES ---\n            max_index = len(test_full_raw) - 1\n            initial_count = len(sorted_indices)\n            sorted_indices = sorted_indices[sorted_indices <= max_index]\n            final_count = len(sorted_indices)\n\n            if initial_count != final_count:\n                print(f\"  Warning: Removed {initial_count - final_count} out-of-bounds indices.\")\n            # --- END OF FIX PART 2 ---\n\n            test_full_raw = test_full_raw.iloc[sorted_indices].copy()\n            test_full_raw.reset_index(drop=True, inplace=True)\n            print(f\"  Test data successfully sorted. New shape: {test_full_raw.shape}\")\n        else:\n            # If the file doesn't exist, raise an error to stop the script.\n            raise FileNotFoundError(f\"CRITICAL: Timestamp reconstruction file not found at {timestamp_recon_path}. Halting execution.\")\n\n        # 3. INITIALIZE AND TRAIN MODEL\n        predictor = CryptoMarketPredictor(top_features=120, top_X_features_to_preselect=30)\n        predictor.fit(train_full_raw)\n\n        # 4. PREDICT\n        predictions, sorted_test_ids = predictor.predict(test_full_raw)\n\n        # 5. ENHANCE WITH PROPHET\n        initial_submission_df = pd.DataFrame({'ID': sorted_test_ids, 'Prediction': predictions})\n        final_submission_df = train_prophet_enhancement(initial_submission_df, prophet_weight=0.085, sample_size=100000)\n\n        # 6. FINALIZE SUBMISSION\n        print(\"\\n\" + \"=\"*50)\n        print(\"PREPARING SUBMISSION\")\n        print(\"=\"*50)\n\n        results_df = pd.DataFrame({'ID': final_submission_df['ID'], 'Prediction': final_submission_df['Prediction']})\n        submission = pd.DataFrame({'ID': original_shuffled_ids})\n        submission = submission.merge(results_df, on='ID', how='left')\n        submission['Prediction'].fillna(0, inplace=True)\n        \n        submission.to_csv('submission.csv', index=False)\n        print(\"Submission saved to 'submission.csv'\")\n        print(\"\\nSubmission preview:\")\n        print(submission.head())\n\n    except Exception as e:\n        print(f\"\\nAn error occurred in the main pipeline: {e}\")\n        traceback.print_exc()\n\n    finally:\n        print(\"\\n🧹 Cleaning up to free memory...\")\n        gc.collect()\n        print(\"Cleanup complete.\")\n\n# --- Run the pipeline ---\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}