{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":96164,"databundleVersionId":11418275,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install hmmlearn","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. Auxiliary Datasets & Feature Creation Notebook:\n2. XGBoost on Seperate Regime Cohorts:\n3. CATBoost on Categorical Regimes:\n4. Meta-Learner Ensemble: ","metadata":{}},{"cell_type":"code","source":"# Suppress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Memory management\nimport gc\nimport psutil\nimport os\n\n# Data manipulation\nimport pandas as pd\nimport numpy as np\n\n# Modeling\nimport xgboost as xgb\nfrom sklearn.decomposition import IncrementalPCA\n\n# Evaluation\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score, roc_auc_score\nfrom sklearn.preprocessing import StandardScaler\n\n# Visualization\nimport matplotlib.pyplot as plt\n\n# Display\nfrom IPython.display import display\nimport time\nimport logging\n\nfrom hmmlearn import hmm\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, confusion_matrix\n\n# Set up logging\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\n\ndef print_memory_usage():\n    \"\"\"Print current memory usage\"\"\"\n    process = psutil.Process(os.getpid())\n    memory_mb = process.memory_info().rss / 1024 / 1024\n    print(f\"💾 Current memory usage: {memory_mb:.2f} MB\")\n    logger.info(f\"Memory usage: {memory_mb:.2f} MB\")\n\ndef check_gpu_availability():\n    \"\"\"Check for GPU availability and return device type\"\"\"\n    try:\n        # Check if XGBoost was compiled with GPU support\n        import xgboost as xgb\n        # Try to create a GPU-enabled booster\n        dtrain = xgb.DMatrix(np.random.rand(10, 5), label=np.random.rand(10))\n        params = {'tree_method': 'gpu_hist', 'gpu_id': 0}\n        bst = xgb.train(params, dtrain, num_boost_round=1, verbose_eval=False)\n        print(\"🚀 GPU detected and will be used for training\")\n        logger.info(\"GPU detected and will be used for training\")\n        return 'gpu'\n    except Exception as e:\n        print(\"⚠️  GPU not available, using CPU with multi-threading\")\n        print(f\"   GPU error: {str(e)}\")\n        logger.warning(f\"GPU not available, using CPU. Error: {str(e)}\")\n        return 'cpu'\n\ndef get_gpu_count():\n    \"\"\"Get number of available GPUs\"\"\"\n    try:\n        import subprocess\n        result = subprocess.run(['nvidia-smi', '--list-gpus'], \n                              capture_output=True, text=True, timeout=5)\n        if result.returncode == 0:\n            gpu_count = len(result.stdout.strip().split('\\n'))\n            print(f\"🎮 Found {gpu_count} GPU(s)\")\n            logger.info(f\"Found {gpu_count} GPU(s)\")\n            return gpu_count\n        else:\n            return 0\n    except:\n        return 0\n\ndef reduce_mem_usage(dataframe, dataset):    \n    print('Reducing memory usage for:', dataset)\n    logger.info(f'Starting memory reduction for: {dataset}')\n    initial_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    \n    for col in dataframe.columns:\n        col_type = dataframe[col].dtype\n\n        c_min = dataframe[col].min()\n        c_max = dataframe[col].max()\n        if str(col_type)[:3] == 'int':\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                dataframe[col] = dataframe[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                dataframe[col] = dataframe[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                dataframe[col] = dataframe[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                dataframe[col] = dataframe[col].astype(np.int64)\n        else:\n            try:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    dataframe[col] = dataframe[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    dataframe[col] = dataframe[col].astype(np.float32)\n                else:\n                    dataframe[col] = dataframe[col].astype(np.float64)\n            except:\n                pass\n\n    final_mem_usage = dataframe.memory_usage().sum() / 1024**2\n    print('--- Memory usage before: {:.2f} MB'.format(initial_mem_usage))\n    print('--- Memory usage after: {:.2f} MB'.format(final_mem_usage))\n    print('--- Decreased memory usage by {:.1f}%\\n'.format(100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage))\n    logger.info(f'Memory reduction for {dataset}: {initial_mem_usage:.2f} MB -> {final_mem_usage:.2f} MB ({100 * (initial_mem_usage - final_mem_usage) / initial_mem_usage:.1f}% reduction)')\n\n    return dataframe\n\ndef check_regime_balance(regimes, max_imbalance=0.4, tolerance=0.1):\n    \"\"\"\n    Check if regimes are reasonably balanced\n    \n    Args:\n        regimes: array of regime predictions\n        max_imbalance: maximum allowed imbalance ratio (minority/majority)\n        tolerance: tolerance for balance check\n    \n    Returns:\n        bool: True if balanced, False otherwise\n    \"\"\"\n    regime_counts = np.bincount(regimes)\n    min_count = regime_counts.min()\n    max_count = regime_counts.max()\n    balance_ratio = min_count / max_count\n    \n    logger.info(f\"Regime balance check: min={min_count}, max={max_count}, ratio={balance_ratio:.3f}\")\n    print(f\"📊 Regime balance: min={min_count}, max={max_count}, ratio={balance_ratio:.3f}\")\n    \n    return balance_ratio >= max_imbalance\n\ndef fit_hmm_with_balance_check(observations, max_attempts=5, n_components_range=(2, 5)):\n    \"\"\"\n    Fit HMM model with balance checking and re-iteration\n    \"\"\"\n    logger.info(\"Starting HMM model fitting with balance checking\")\n    print(\"🎯 Starting HMM model fitting with balance checking...\")\n    \n    best_model = None\n    best_balance_ratio = 0\n    best_regimes = None\n    \n    for attempt in range(max_attempts):\n        logger.info(f\"HMM fitting attempt {attempt + 1}/{max_attempts}\")\n        print(f\"🔄 Attempt {attempt + 1}/{max_attempts}\")\n        \n        for n_components in range(n_components_range[0], n_components_range[1] + 1):\n            logger.info(f\"Trying HMM with {n_components} components\")\n            print(f\"   📈 Testing {n_components} regimes...\")\n            \n            try:\n                # Create model with different parameters each attempt\n                model = hmm.GaussianHMM(\n                    n_components=n_components,\n                    covariance_type=\"diag\" if attempt % 2 == 0 else \"spherical\",\n                    n_iter=200 + attempt * 50,\n                    tol=1e-4,\n                    random_state=42 + attempt * 10,\n                    init_params=\"stmc\"\n                )\n                \n                logger.info(f\"Fitting HMM model with {n_components} components...\")\n                print(f\"      🔧 Fitting model...\")\n                model.fit(observations)\n                \n                # Predict regimes\n                predicted_regimes = model.predict(observations)\n                \n                # Check balance\n                regime_counts = np.bincount(predicted_regimes)\n                min_count = regime_counts.min()\n                max_count = regime_counts.max()\n                balance_ratio = min_count / max_count\n                \n                logger.info(f\"Model with {n_components} components: balance ratio = {balance_ratio:.3f}\")\n                print(f\"      📊 Balance ratio: {balance_ratio:.3f}\")\n                \n                # Update best model if this one is more balanced\n                if balance_ratio > best_balance_ratio:\n                    best_balance_ratio = balance_ratio\n                    best_model = model\n                    best_regimes = predicted_regimes\n                    logger.info(f\"New best model found with balance ratio: {balance_ratio:.3f}\")\n                    print(f\"      ✅ New best model! Balance: {balance_ratio:.3f}\")\n                \n                # If we found a well-balanced model, use it\n                if check_regime_balance(predicted_regimes):\n                    logger.info(f\"Found well-balanced model with {n_components} components\")\n                    print(f\"      🎉 Well-balanced model found!\")\n                    return model, predicted_regimes\n                    \n            except Exception as e:\n                logger.warning(f\"HMM fitting failed for {n_components} components: {e}\")\n                print(f\"      ⚠️ Failed with {n_components} components: {e}\")\n                continue\n    \n    # Return best model found\n    if best_model is not None:\n        logger.info(f\"Returning best model with balance ratio: {best_balance_ratio:.3f}\")\n        print(f\"🏆 Using best model found with balance ratio: {best_balance_ratio:.3f}\")\n        return best_model, best_regimes\n    else:\n        raise ValueError(\"Failed to fit any HMM model\")\n\ndef fit_classifier_with_balance_check(X_train, y_train, max_attempts=5):\n    \"\"\"\n    Fit classifier with balance checking and re-iteration\n    \"\"\"\n    logger.info(\"Starting classifier fitting with balance checking\")\n    print(\"🎯 Starting classifier model fitting with balance checking...\")\n    \n    best_clf = None\n    best_auc = 0\n    best_balance_ratio = 0\n    \n    for attempt in range(max_attempts):\n        logger.info(f\"Classifier fitting attempt {attempt + 1}/{max_attempts}\")\n        print(f\"🔄 Classifier attempt {attempt + 1}/{max_attempts}\")\n        \n        try:\n            # Try different classifier parameters\n            clf = RandomForestClassifier(\n                n_estimators=100 + attempt * 50,\n                max_depth=10 + attempt * 2,\n                min_samples_split=2 + attempt,\n                random_state=42 + attempt * 10,\n                class_weight='balanced' if attempt > 2 else None\n            )\n            \n            logger.info(\"Fitting Random Forest classifier...\")\n            print(\"   🌲 Fitting Random Forest...\")\n            clf.fit(X_train, y_train)\n            \n            # Predict and evaluate\n            train_preds = clf.predict(X_train)\n            \n            # Check balance of predictions\n            pred_counts = np.bincount(train_preds)\n            min_count = pred_counts.min()\n            max_count = pred_counts.max()\n            balance_ratio = min_count / max_count\n            \n            # Calculate ROC AUC (for multiclass, use ovr strategy)\n            try:\n                if len(np.unique(y_train)) > 2:\n                    train_proba = clf.predict_proba(X_train)\n                    auc_score = roc_auc_score(y_train, train_proba, multi_class='ovr', average='weighted')\n                else:\n                    train_proba = clf.predict_proba(X_train)[:, 1]\n                    auc_score = roc_auc_score(y_train, train_proba)\n            except Exception as e:\n                logger.warning(f\"Could not calculate ROC AUC: {e}\")\n                auc_score = 0\n            \n            logger.info(f\"Classifier attempt {attempt + 1}: AUC={auc_score:.4f}, balance={balance_ratio:.3f}\")\n            print(f\"   📊 AUC: {auc_score:.4f}, Balance: {balance_ratio:.3f}\")\n            \n            # Update best classifier based on both AUC and balance\n            combined_score = auc_score * 0.7 + balance_ratio * 0.3\n            best_combined = best_auc * 0.7 + best_balance_ratio * 0.3\n            \n            if combined_score > best_combined:\n                best_clf = clf\n                best_auc = auc_score\n                best_balance_ratio = balance_ratio\n                logger.info(f\"New best classifier found: AUC={auc_score:.4f}, balance={balance_ratio:.3f}\")\n                print(f\"   ✅ New best classifier!\")\n            \n            # If we have good balance and AUC, stop\n            if balance_ratio >= 0.3 and auc_score >= 0.7:\n                logger.info(\"Found satisfactory classifier with good balance and AUC\")\n                print(f\"   🎉 Satisfactory classifier found!\")\n                return clf, auc_score, balance_ratio\n                \n        except Exception as e:\n            logger.warning(f\"Classifier fitting failed in attempt {attempt + 1}: {e}\")\n            print(f\"   ⚠️ Attempt {attempt + 1} failed: {e}\")\n            continue\n    \n    if best_clf is not None:\n        logger.info(f\"Returning best classifier: AUC={best_auc:.4f}, balance={best_balance_ratio:.3f}\")\n        print(f\"🏆 Using best classifier found: AUC={best_auc:.4f}, balance={best_balance_ratio:.3f}\")\n        return best_clf, best_auc, best_balance_ratio\n    else:\n        raise ValueError(\"Failed to fit any classifier\")\n\ndef add_features(df):\n    data = df.copy()\n    features_df = pd.DataFrame(index=data.index)\n    \n    features_df['bid_ask_spread_proxy'] = data['ask_qty'] - data['bid_qty']\n    features_df['total_liquidity'] = data['bid_qty'] + data['ask_qty']\n    features_df['trade_imbalance'] = data['buy_qty'] - data['sell_qty']\n    features_df['total_trades'] = data['buy_qty'] + data['sell_qty']\n    \n    features_df['volume_per_trade'] = data['volume'] / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n    features_df['buy_volume_ratio'] = data['buy_qty'] / (data['volume'] + 1e-8)\n    features_df['sell_volume_ratio'] = data['sell_qty'] / (data['volume'] + 1e-8)\n    \n    features_df['buying_pressure'] = data['buy_qty'] / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n    features_df['selling_pressure'] = data['sell_qty'] / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n    \n    features_df['order_imbalance'] = (data['bid_qty'] - data['ask_qty']) / (data['bid_qty'] + data['ask_qty'] + 1e-8)\n    features_df['order_imbalance_abs'] = np.abs(features_df['order_imbalance'])\n    features_df['bid_liquidity_ratio'] = data['bid_qty'] / (data['volume'] + 1e-8)\n    features_df['ask_liquidity_ratio'] = data['ask_qty'] / (data['volume'] + 1e-8)\n    features_df['depth_imbalance'] = features_df['total_trades'] - data['volume']\n    \n    features_df['buy_sell_ratio'] = data['buy_qty'] / (data['sell_qty'] + 1e-8)\n    features_df['bid_ask_ratio'] = data['bid_qty'] / (data['ask_qty'] + 1e-8)\n    features_df['volume_liquidity_ratio'] = data['volume'] / (data['bid_qty'] + data['ask_qty'] + 1e-8)\n\n    features_df['buy_volume_product'] = data['buy_qty'] * data['volume']\n    features_df['sell_volume_product'] = data['sell_qty'] * data['volume']\n    features_df['bid_ask_product'] = data['bid_qty'] * data['ask_qty']\n    \n    features_df['market_competition'] = (data['buy_qty'] * data['sell_qty']) / ((data['buy_qty'] + data['sell_qty']) + 1e-8)\n    features_df['liquidity_competition'] = (data['bid_qty'] * data['ask_qty']) / ((data['bid_qty'] + data['ask_qty']) + 1e-8)\n    \n    total_activity = data['buy_qty'] + data['sell_qty'] + data['bid_qty'] + data['ask_qty']\n    features_df['market_activity'] = total_activity\n    features_df['activity_concentration'] = data['volume'] / (total_activity + 1e-8)\n    \n    features_df['info_arrival_rate'] = (data['buy_qty'] + data['sell_qty']) / (data['volume'] + 1e-8)\n    features_df['market_making_intensity'] = (data['bid_qty'] + data['ask_qty']) / (data['buy_qty'] + data['sell_qty'] + 1e-8)\n    features_df['effective_spread_proxy'] = np.abs(data['buy_qty'] - data['sell_qty']) / (data['volume'] + 1e-8)\n    \n    lambda_decay = 0.95\n    ofi = data['buy_qty'] - data['sell_qty']\n    features_df['order_flow_imbalance_ewm'] = ofi.ewm(alpha=1-lambda_decay).mean()\n\n    features_df = features_df.replace([np.inf, -np.inf], np.nan)\n    \n    return features_df\n\ndef select_top_k_features_fast(X_data, y_data, feature_columns, k):\n    \"\"\"\n    Fast in-memory feature selection based on absolute correlation with target.\n    \n    Args:\n        X_data: DataFrame with features\n        y_data: Series with target values  \n        feature_columns: List of feature column names\n        k: Number of top features to select\n    \n    Returns:\n        list: Top k feature names\n    \"\"\"\n    logger.info(f\"Starting fast feature selection: selecting top {k} features\")\n    print(f\"🚀 Fast feature selection: selecting top {k} features...\")\n    \n    # Ensure all features are numeric\n    print(\"   🔧 Ensuring numeric data types...\")\n    X_numeric = X_data[feature_columns].copy()\n    \n    # Convert to numeric, coercing errors to NaN\n    for col in feature_columns:\n        X_numeric[col] = pd.to_numeric(X_numeric[col], errors='coerce')\n    \n    # Fill any remaining NaN with 0\n    X_numeric = X_numeric.fillna(0)\n    \n    # Calculate correlations using pandas (much faster)\n    print(\"   📊 Computing correlations with target...\")\n    logger.info(\"Computing correlations with vectorized operations\")\n    \n    correlations = {}\n    y_numeric = pd.to_numeric(y_data, errors='coerce').fillna(0)\n    \n    # Use pandas correlation which is optimized\n    correlation_series = X_numeric.corrwith(y_numeric)\n    \n    # Convert to absolute values and handle NaN\n    for feature in feature_columns:\n        corr_val = correlation_series.get(feature, 0.0)\n        if pd.isna(corr_val):\n            corr_val = 0.0\n        correlations[feature] = abs(corr_val)\n    \n    # Select top k features\n    print(\"   🎯 Selecting top features...\")\n    top_features = sorted(correlations.keys(), key=lambda x: correlations[x], reverse=True)[:k]\n    \n    logger.info(f\"Feature selection completed: selected {len(top_features)} features\")\n    print(f\"   ✅ Selected {len(top_features)} features\")\n    \n    # Log top 10 correlations for debugging\n    top_10_correlations = [(feat, correlations[feat]) for feat in top_features[:10]]\n    logger.info(f\"Top 10 feature correlations: {top_10_correlations}\")\n    print(f\"   📈 Top 5 correlations: {[(f, f'{c:.4f}') for f, c in top_10_correlations[:5]]}\")\n    \n    return top_features\n\ndef create_component_features(input_file, output_file, target_col, feature_columns=None, n_components=5, chunksize=50000):\n    \"\"\"\n    Creates principal component features from a large dataset and writes the result to a CSV.\n    Only saves timestamp/ID, target column (if present), and PCA components.\n    Optimized version with faster processing and better memory management.\n    \n    Args:\n        input_file (str): Path to input CSV file.\n        output_file (str): Path to output CSV file.\n        target_col (str): Name of the target column.\n        feature_columns (list): List of feature column names to use for PCA. If None, uses all columns except target.\n        n_components (int): Number of principal components to create (default: 5).\n        chunksize (int): Number of rows per processing chunk (default: 50000).\n    \"\"\"\n    logger.info(f\"Starting optimized PCA component creation: {n_components} components from {input_file}\")\n    print(f\"\\n🚀 Creating {n_components} PCA components (optimized version)...\")\n    \n    # Initialize memory tracking\n    process = psutil.Process(os.getpid())\n    initial_memory = process.memory_info().rss / 1024 / 1024\n    print(f\"   💾 Initial memory usage: {initial_memory:.1f} MB\")\n    \n    # First pass: Determine feature columns and compute statistics\n    print(\"   📊 Pass 1/3: Computing global statistics...\")\n    logger.info(\"PCA Pass 1: Computing global statistics\")\n    \n    first_chunk = True\n    n_samples = 0\n    running_mean = None\n    running_var = None\n    chunk_count = 0\n    total_chunks = sum(1 for _ in pd.read_csv(input_file, chunksize=chunksize))\n    id_col = None  # To store ID column name if present\n    \n    print(f\"      Total chunks to process: {total_chunks}\")\n    \n    for chunk in pd.read_csv(input_file, chunksize=chunksize):\n        chunk_count += 1\n        \n        if first_chunk:\n            # Determine ID column (either 'timestamp' or 'ID')\n            if 'timestamp' in chunk.columns:\n                id_col = 'timestamp'\n            elif 'ID' in chunk.columns:\n                id_col = 'ID'\n            \n            if feature_columns is None:\n                # Exclude target and ID columns from features\n                exclude_cols = [target_col] if target_col in chunk.columns else []\n                if id_col:\n                    exclude_cols.append(id_col)\n                feature_columns = [col for col in chunk.columns if col not in exclude_cols]\n            \n            n_features = len(feature_columns)\n            running_mean = np.zeros(n_features)\n            running_var = np.zeros(n_features)\n            print(f\"      Using {n_features} features\")\n            print(f\"      ID column: {id_col}\")\n            print(f\"      Target column: {target_col if target_col in chunk.columns else 'Not present'}\")\n            logger.info(f\"Using {n_features} features for PCA, ID column: {id_col}\")\n            first_chunk = False\n        \n        if chunk_count % 5 == 0:\n            progress = (chunk_count / total_chunks) * 100\n            print(f\"      ⏳ Progress: {progress:.1f}% (chunk {chunk_count}/{total_chunks})\")\n            logger.info(f\"Pass 1: Processing chunk {chunk_count}/{total_chunks}\")\n        \n        X_chunk = chunk[feature_columns].values.astype(np.float32)  # Use float32 for memory efficiency\n        chunk_size = X_chunk.shape[0]\n        \n        # Update running statistics using Welford's online algorithm\n        if n_samples == 0:\n            running_mean = np.mean(X_chunk, axis=0)\n            running_var = np.var(X_chunk, axis=0)\n        else:\n            chunk_mean = np.mean(X_chunk, axis=0)\n            chunk_var = np.var(X_chunk, axis=0)\n            delta = chunk_mean - running_mean\n            running_mean += delta * chunk_size / (n_samples + chunk_size)\n            running_var = (n_samples * running_var + chunk_size * chunk_var + \n                         delta * delta * n_samples * chunk_size / (n_samples + chunk_size)) / (n_samples + chunk_size)\n        \n        n_samples += chunk_size\n        \n        # Clear memory\n        del X_chunk\n        gc.collect()\n    \n    global_std = np.sqrt(running_var)\n    global_std[global_std == 0] = 1.0\n    \n    current_memory = process.memory_info().rss / 1024 / 1024\n    print(f\"      💾 Memory usage after Pass 1: {current_memory:.1f} MB\")\n    print(f\"      ✅ Pass 1 completed: {n_samples:,} samples processed\")\n    \n    # Initialize Incremental PCA with reduced components\n    print(\"   🔧 Pass 2/3: Training PCA model...\")\n    logger.info(\"PCA Pass 2: Training Incremental PCA\")\n    \n    ipca = IncrementalPCA(n_components=n_components)\n    chunk_count = 0\n    \n    for chunk in pd.read_csv(input_file, chunksize=chunksize):\n        chunk_count += 1\n        if chunk_count % 5 == 0:\n            progress = (chunk_count / total_chunks) * 100\n            print(f\"      ⏳ Progress: {progress:.1f}% (chunk {chunk_count}/{total_chunks})\")\n            logger.info(f\"Pass 2: Processing chunk {chunk_count}/{total_chunks}\")\n        \n        X_chunk = chunk[feature_columns].values.astype(np.float32)\n        X_scaled = (X_chunk - running_mean) / global_std\n        ipca.partial_fit(X_scaled)\n        \n        del X_chunk, X_scaled\n        gc.collect()\n    \n    explained_var = [f\"{var:.3f}\" for var in ipca.explained_variance_ratio_]\n    print(f\"      📈 Explained variance ratios: {explained_var}\")\n    logger.info(f\"PCA components explain: {explained_var} of variance\")\n    \n    current_memory = process.memory_info().rss / 1024 / 1024\n    print(f\"      💾 Memory usage after Pass 2: {current_memory:.1f} MB\")\n    \n    # Third pass: Transform data and write to output\n    print(\"   💾 Pass 3/3: Transforming data and saving...\")\n    logger.info(\"PCA Pass 3: Transforming and saving data\")\n    \n    first_chunk = True\n    chunk_count = 0\n    \n    for chunk in pd.read_csv(input_file, chunksize=chunksize):\n        chunk_count += 1\n        if chunk_count % 5 == 0:\n            progress = (chunk_count / total_chunks) * 100\n            print(f\"      ⏳ Progress: {progress:.1f}% (chunk {chunk_count}/{total_chunks})\")\n            logger.info(f\"Pass 3: Processing chunk {chunk_count}/{total_chunks}\")\n        \n        X_chunk = chunk[feature_columns].values.astype(np.float32)\n        X_scaled = (X_chunk - running_mean) / global_std\n        components = ipca.transform(X_scaled)\n        \n        comp_cols = [f'pca_{i}' for i in range(components.shape[1])]\n        components_df = pd.DataFrame(components, columns=comp_cols)\n        \n        # Create output with only ID/timestamp, target (if present), and PCA components\n        output_columns = []\n        output_chunk = pd.DataFrame()\n        \n        # Add ID column if present\n        if id_col and id_col in chunk.columns:\n            output_chunk[id_col] = chunk[id_col]\n            output_columns.append(id_col)\n        \n        # Add target column if present\n        if target_col in chunk.columns:\n            output_chunk[target_col] = chunk[target_col]\n            output_columns.append(target_col)\n        \n        # Add PCA components\n        output_chunk = pd.concat([output_chunk, components_df], axis=1)\n        \n        output_chunk.to_csv(output_file, mode='a', header=first_chunk, index=False)\n        first_chunk = False\n        \n        del X_chunk, X_scaled, components, components_df, output_chunk\n        gc.collect()\n    \n    final_memory = process.memory_info().rss / 1024 / 1024\n    memory_change = final_memory - initial_memory\n    print(f\"\\n   💾 Final memory usage: {final_memory:.1f} MB (change: {memory_change:+.1f} MB)\")\n    print(f\"   ✅ PCA components saved to {output_file}\")\n    print(f\"   📊 Output contains: {id_col if id_col else 'No ID'}, {'target' if target_col else 'No target'}, {n_components} PCA components\")\n    logger.info(f\"PCA completed: {n_components} components saved to {output_file}\")\n\n# Check system resources\nprint(\"🔍 Checking system resources...\")\nlogger.info(\"Starting system resource check\")\ndevice_type = check_gpu_availability()\ncpu_count = os.cpu_count()\ngpu_count = get_gpu_count() if device_type == 'gpu' else 0\nprint(f\"💻 Available CPU cores: {cpu_count}\")\nlogger.info(f\"Available CPU cores: {cpu_count}\")\nif gpu_count > 0:\n    print(f\"🎮 Available GPUs: {gpu_count}\")\n    logger.info(f\"Available GPUs: {gpu_count}\")\nprint_memory_usage()\nprint()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 0: Load Data and Optimize dtypes for lesser memory usage ---","metadata":{}},{"cell_type":"code","source":"logger.info(\"Loading and optimizing data...\")\nprint(\"📂 Loading data...\")\ntrain_path = '/kaggle/input/drw-crypto-market-prediction/train.parquet'\ntest_path = '/kaggle/input/drw-crypto-market-prediction/test.parquet'\ntrain = pd.read_parquet(train_path).reset_index(drop=False)\ntest = pd.read_parquet(test_path).reset_index(drop=False)\nlogger.info(f\"Loaded train data: {train.shape}, test data: {test.shape}\")\ntrain = reduce_mem_usage(train, \"train\")\ntest = reduce_mem_usage(test, \"test\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 1: Create X_train, y_train, X_test ---","metadata":{}},{"cell_type":"code","source":"logger.info(\"Preparing training and test sets...\")\nprint(\"🎯 Preparing training and test sets...\")\ntarget = 'label'\nX_train = train.copy()\ny_train = train[target]\nX_test = test.drop(target, axis=1)\ndel train,test\nprint(f\"Train data: {len(X_train)} samples\")\nprint(f\"Test data: {len(X_test)} samples\")\nlogger.info(f\"Train samples: {len(X_train)}, Test samples: {len(X_test)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 2: Add extra derived features & Popular features in discussions---","metadata":{}},{"cell_type":"code","source":"logger.info(\"Adding derived features...\")\nprint(\"🔧 Adding derived features...\")\nX_train = pd.concat([add_features(X_train), X_train], axis=1)\nX_test = pd.concat([add_features(X_test), X_test], axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 2.5: Create Enhanced Features with PCA and Feature Selection ---","metadata":{}},{"cell_type":"code","source":"logger.info(\"Creating enhanced features with PCA and feature selection...\")\nprint(\"🚀 Creating two enhanced datasets: PCA components and Top 500 features...\")\n\n\n\n\n\n\n\n# Define feature lists for regime detection\nregimeFeatures_HMM = ['info_arrival_rate', 'market_making_intensity', 'effective_spread_proxy', 'order_flow_imbalance_ewm']\nregimeFeatures_CLF = ['bid_qty', 'ask_qty', 'buy_qty', 'sell_qty', 'volume', 'trade_imbalance']\n\n# Get all available features (excluding timestamp/ID and target)\nall_features = [col for col in X_train.columns if col not in ['timestamp', target]]\nprint(f\"   📊 Total available features: {len(all_features)}\")\nlogger.info(f\"Total available features for processing: {len(all_features)}\")\n\n# --- Comprehensive Data Cleaning ---\nprint(\"\\n🧹 Comprehensive data cleaning...\")\nlogger.info(\"Starting comprehensive data cleaning\")\n\n# Check for NaN and infinite values in training data\nprint(\"   🔍 Checking training data...\")\ntrain_nan_count = X_train[all_features].isna().sum().sum()\ntrain_inf_count = np.isinf(X_train[all_features].select_dtypes(include=[np.number]).values).sum()\nprint(f\"      - Training NaN values: {train_nan_count}\")\nprint(f\"      - Training infinite values: {train_inf_count}\")\nlogger.info(f\"Training data: {train_nan_count} NaN, {train_inf_count} infinite values\")\n\n# Check for NaN and infinite values in test data\nprint(\"   🔍 Checking test data...\")\ntest_nan_count = X_test[all_features].isna().sum().sum()\ntest_inf_count = np.isinf(X_test[all_features].select_dtypes(include=[np.number]).values).sum()\nprint(f\"      - Test NaN values: {test_nan_count}\")\nprint(f\"      - Test infinite values: {test_inf_count}\")\nlogger.info(f\"Test data: {test_nan_count} NaN, {test_inf_count} infinite values\")\n\n# Clean training data\nprint(\"   🔧 Cleaning training data...\")\n# Replace infinite values with NaN first\nX_train[all_features] = X_train[all_features].replace([np.inf, -np.inf], np.nan)\n\n# Fill NaN values with column medians (more robust than mean)\nfor col in all_features:\n    if X_train[col].isna().any():\n        median_val = X_train[col].median()\n        if pd.isna(median_val):  # If median is also NaN, use 0\n            median_val = 0.0\n        X_train[col] = X_train[col].fillna(median_val)\n        logger.info(f\"Filled NaN in training column {col} with {median_val}\")\n\n# Clean test data using training data statistics\nprint(\"   🔧 Cleaning test data...\")\n# Replace infinite values with NaN first\nX_test[all_features] = X_test[all_features].replace([np.inf, -np.inf], np.nan)\n\n# Fill NaN values with corresponding training medians\nfor col in all_features:\n    if X_test[col].isna().any():\n        # Use the same median from training data for consistency\n        median_val = X_train[col].median()\n        if pd.isna(median_val):  # If median is still NaN, use 0\n            median_val = 0.0\n        X_test[col] = X_test[col].fillna(median_val)\n        logger.info(f\"Filled NaN in test column {col} with training median {median_val}\")\n\n# Final verification\nprint(\"   ✅ Final verification...\")\nfinal_train_nan = X_train[all_features].isna().sum().sum()\nfinal_train_inf = np.isinf(X_train[all_features].select_dtypes(include=[np.number]).values).sum()\nfinal_test_nan = X_test[all_features].isna().sum().sum()\nfinal_test_inf = np.isinf(X_test[all_features].select_dtypes(include=[np.number]).values).sum()\n\nprint(f\"      - Final training NaN: {final_train_nan}, infinite: {final_train_inf}\")\nprint(f\"      - Final test NaN: {final_test_nan}, infinite: {final_test_inf}\")\nlogger.info(f\"Final cleanup - Train: {final_train_nan} NaN, {final_train_inf} inf; Test: {final_test_nan} NaN, {final_test_inf} inf\")\n\nif final_train_nan > 0 or final_train_inf > 0 or final_test_nan > 0 or final_test_inf > 0:\n    print(\"      ⚠️ Warning: Still have NaN/infinite values, applying final fallback...\")\n    # Fallback: replace any remaining NaN/inf with 0\n    X_train[all_features] = X_train[all_features].fillna(0).replace([np.inf, -np.inf], 0)\n    X_test[all_features] = X_test[all_features].fillna(0).replace([np.inf, -np.inf], 0)\n    logger.warning(\"Applied fallback: replaced remaining NaN/inf with 0\")\n\nprint(\"   🎉 Data cleaning completed!\")\nlogger.info(\"Data cleaning completed successfully\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Dataset 1: PCA Components Dataset ---","metadata":{}},{"cell_type":"code","source":"print(\"\\n🔧 Creating Dataset 1: PCA Components...\")\nlogger.info(\"Creating PCA components dataset\")\n\n# Define feature columns for PCA (exclude timestamp and target)\npca_feature_columns = all_features\nlogger.info(f\"PCA will use {len(pca_feature_columns)} feature columns\")\nprint(f\"   🎯 Using {len(pca_feature_columns)} features for PCA\")\n\n# Prepare data for PCA (directly from memory)\nprint(\"   🔧 Preparing data for PCA processing...\")\nX_train_features = X_train[pca_feature_columns].values.astype(np.float32)\nX_test_features = X_test[pca_feature_columns].values.astype(np.float32)\n\n# Fit and transform with Incremental PCA\nprint(\"   🔧 Fitting PCA model...\")\nlogger.info(\"Fitting Incremental PCA on training data\")\nipca = IncrementalPCA(n_components=20)\n\n# Fit PCA on training data in chunks if needed\nchunk_size = 10000\nn_samples = X_train_features.shape[0]\nfor i in range(0, n_samples, chunk_size):\n    end_idx = min(i + chunk_size, n_samples)\n    chunk = X_train_features[i:end_idx]\n    ipca.partial_fit(chunk)\n\n# Transform both datasets\nprint(\"   🔄 Transforming datasets with PCA...\")\nX_train_pca_components = ipca.transform(X_train_features)\nX_test_pca_components = ipca.transform(X_test_features)\n\n# Create PCA component column names\npca_component_names = [f'pca_{i}' for i in range(20)]\nexplained_var = [f\"{var:.3f}\" for var in ipca.explained_variance_ratio_]\nprint(f\"   📈 Explained variance ratios: {explained_var}\")\nlogger.info(f\"PCA components explain: {explained_var} of variance\")\n\n# Create final PCA datasets\nprint(\"   💾 Creating PCA datasets...\")\npca_train_file = 'pca_train_dataset.csv'\npca_test_file = 'pca_test_dataset.csv'\n\n# Train PCA dataset: timestamp + target + PCA components\npca_train_df = pd.DataFrame(X_train_pca_components, columns=pca_component_names)\npca_train_df.insert(0, 'timestamp', X_train['timestamp'].values)\npca_train_df.insert(1, target, X_train[target].values)\npca_train_df.to_csv(pca_train_file, index=False)\n\n# Test PCA dataset: ID + PCA components\npca_test_df = pd.DataFrame(X_test_pca_components, columns=pca_component_names)\npca_test_df.insert(0, 'ID', X_test['ID'].values)\npca_test_df.to_csv(pca_test_file, index=False)\n\nprint(f\"   📊 PCA train dataset: {pca_train_df.shape}\")\nprint(f\"   📊 PCA test dataset: {pca_test_df.shape}\")\nprint(f\"   ✅ PCA datasets saved: {pca_train_file}, {pca_test_file}\")\nlogger.info(f\"PCA datasets created - Train: {pca_train_df.shape}, Test: {pca_test_df.shape}\")\n\n# Clean up PCA variables\ndel X_train_features, X_test_features, X_train_pca_components, X_test_pca_components, pca_train_df, pca_test_df\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Dataset 2: Top 500 Features Dataset ---","metadata":{}},{"cell_type":"code","source":"print(\"\\n🎯 Creating Dataset 2: Top 500 Features...\")\nlogger.info(\"Creating top 500 features dataset\")\n\n# Select top 500 features from all available features (directly from memory)\nprint(f\"   📊 Selecting top 500 from {len(all_features)} available features...\")\n\n# Use fast in-memory feature selection\ntop_500_features = select_top_k_features_fast(\n    X_data=X_train, \n    y_data=y_train, \n    feature_columns=all_features,\n    k=min(500, len(all_features))\n)\n\nprint(f\"   ✅ Selected {len(top_500_features)} features (showing first 5): {top_500_features[:5]}\")\nlogger.info(f\"Selected {len(top_500_features)} top features\")\n\n# Create top 500 features datasets (directly from memory)\ntop500_train_file = 'top500_train_dataset.csv'\ntop500_test_file = 'top500_test_dataset.csv'\n\nprint(\"   💾 Creating top 500 features datasets...\")\n\n# For train data: timestamp, target, top 500 features\ntrain_top500_columns = ['timestamp', target] + top_500_features\nX_train_top500 = X_train[train_top500_columns]\nX_train_top500.to_csv(top500_train_file, index=False)\n\n# For test data: ID, top 500 features\ntest_top500_columns = ['ID'] + top_500_features\nX_test_top500 = X_test[test_top500_columns]\nX_test_top500.to_csv(top500_test_file, index=False)\n\nprint(f\"   📊 Top 500 train dataset: {X_train_top500.shape}\")\nprint(f\"   📊 Top 500 test dataset: {X_test_top500.shape}\")\nlogger.info(f\"Top 500 datasets created - Train: {X_train_top500.shape}, Test: {X_test_top500.shape}\")\n\n# --- Memory Cleanup ---\nprint(\"\\n🧹 Cleaning up memory...\")\nlogger.info(\"Starting memory cleanup\")\n\n# Delete large intermediate dataframes\ndel X_train_top500, X_test_top500\ngc.collect()\n\nprint_memory_usage()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 3: Load Final Datasets for Processing & Regime Detection Using HMM---","metadata":{}},{"cell_type":"code","source":"\nprint(\"\\n📂 Loading final datasets for regime detection...\")\nlogger.info(\"Loading final datasets for regime detection\")\n\n# Load PCA dataset for HMM training (will use PCA components + some original features)\nprint(\"   📂 Loading PCA dataset for HMM...\")\nX_train_pca_hmm = pd.read_csv(pca_train_file)\nlogger.info(f\"Loaded PCA dataset for HMM: {X_train_pca_hmm.shape}\")\n\n# Load top 500 dataset for classifier training\nprint(\"   📂 Loading top 500 training dataset...\")\nX_train_top500_loaded = pd.read_csv(top500_train_file)\nlogger.info(f\"Loaded top 500 train dataset: {X_train_top500_loaded.shape}\")\n\n# Update regime features for classifier (use original classifier features from top 500)\nregimeFeatures_CLF_enhanced = [f for f in regimeFeatures_CLF if f in X_train_top500_loaded.columns]\n\nprint(f\"   🌲 Classifier features: {len(regimeFeatures_CLF_enhanced)} original features\")\nlogger.info(f\"Classifier feature set: {len(regimeFeatures_CLF_enhanced)} features\")\n\n# Clean up original large dataframes\ndel X_train, X_test\ngc.collect()\n\nprint(\"   ✅ Memory cleanup completed\")\nlogger.info(\"Memory cleanup completed\")\nprint_memory_usage()\n\n# Use the loaded datasets for further processing\nX_train = X_train_top500_loaded  # Use top 500 dataset as main training data\ny_train = X_train[target]\n\n# Extract PCA components and target for observations\npca_features_for_hmm = [f'pca_{i}' for i in range(20)]\ny_train_pca = X_train_pca_hmm[target]\n\nprint(f\"   🎯 Using {len(pca_features_for_hmm)} PCA components for HMM regime detection\")\nlogger.info(f\"Using {len(pca_features_for_hmm)} PCA components for HMM\")\n\n# Prepare observation sequence (PCA features + returns)\nobservations = pd.concat([X_train_pca_hmm[pca_features_for_hmm], y_train_pca], axis=1)\n\n# Clean observations data to handle NaN and infinite values\nprint(\"🧹 Cleaning observations data for HMM...\")\nlogger.info(\"Cleaning observations data for HMM\")\nprint(f\"   - Original shape: {observations.shape}\")\n\n# Check for NaN and infinite values\nnan_count = observations.isna().sum().sum()\ninf_count = np.isinf(observations.values).sum()\nprint(f\"   - NaN values found: {nan_count}\")\nprint(f\"   - Infinite values found: {inf_count}\")\nlogger.info(f\"Data cleaning: {nan_count} NaN values, {inf_count} infinite values\")\n\n# Replace infinite values with NaN first\nobservations = observations.replace([np.inf, -np.inf], np.nan)\n\n# Fill NaN values with column means\nobservations = observations.fillna(observations.mean())\n\n# Final check and fallback to zeros if mean is still NaN\nobservations = observations.fillna(0)\n\n# Verify data is clean\nfinal_nan_count = observations.isna().sum().sum()\nfinal_inf_count = np.isinf(observations.values).sum()\nprint(f\"   - Final NaN count: {final_nan_count}\")\nprint(f\"   - Final infinite count: {final_inf_count}\")\nprint(f\"   - Clean shape: {observations.shape}\")\nlogger.info(f\"Data cleaned: {final_nan_count} NaN, {final_inf_count} infinite values remaining\")\n\n# Standardize the data to prevent numerical issues\nprint(\"🔧 Standardizing observations for numerical stability...\")\nlogger.info(\"Standardizing observations for HMM\")\nscaler = StandardScaler()\nobservations_scaled = scaler.fit_transform(observations)\n\n# Add small noise to prevent singular covariance matrices\nprint(\"🎲 Adding small regularization noise...\")\nlogger.info(\"Adding regularization noise to prevent singular matrices\")\nnp.random.seed(42)\nregularization_noise = np.random.normal(0, 1e-6, observations_scaled.shape)\nobservations_scaled += regularization_noise\n\n# Convert to numpy array\nobservations = observations_scaled\n\nprint(f\"   - Standardized data stats: mean={observations.mean():.6f}, std={observations.std():.6f}\")\nprint(f\"   - Data range: [{observations.min():.6f}, {observations.max():.6f}]\")\nlogger.info(f\"Standardized data: mean={observations.mean():.6f}, std={observations.std():.6f}\")\n\n# Fit Gaussian HMM with balance checking\nlogger.info(\"Fitting HMM model with balance checking...\")\nmodel, predicted_regimes = fit_hmm_with_balance_check(observations)\n\n# Add predicted regimes to both datasets (PCA and top500)\nX_train_pca_hmm['predicted_regime'] = predicted_regimes\nX_train['predicted_regime'] = predicted_regimes  # Also add to main training dataset\n\nprint(f\"📊 Final regime distribution: {np.bincount(predicted_regimes)}\")\nprint(f\"📈 Number of regimes detected: {len(np.unique(predicted_regimes))}\")\nlogger.info(f\"HMM regime distribution: {dict(enumerate(np.bincount(predicted_regimes)))}\")\n\n# --- Step 3.5: Analyze Regime Properties ---\nlogger.info(\"Analyzing regime properties...\")\nprint(\"📈 Analyzing regime properties...\")\n\n# Map HMM states to consistent regime numbering using PCA dataset\nregime_mapping = {}\nfor regime_id in range(len(np.unique(predicted_regimes))):\n    regime_data = X_train_pca_hmm[X_train_pca_hmm['predicted_regime'] == regime_id]\n    mean_return = regime_data[target].mean()\n    std_return = regime_data[target].std()\n    duration = len(regime_data)\n    regime_mapping[regime_id] = (mean_return, std_return, duration)\n\nprint(\"\\nDetected Regime Properties:\")\nlogger.info(\"Regime properties analysis:\")\nfor regime_id, (mean, std, dur) in regime_mapping.items():\n    print(f\"Regime {regime_id}: μ={mean:.6f}, σ={std:.6f}, duration={dur}s\")\n    logger.info(f\"Regime {regime_id}: μ={mean:.6f}, σ={std:.6f}, duration={dur}s\")\n\n# Clean up PCA HMM dataset to save memory\ndel X_train_pca_hmm\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 4: Train Regime Classifier ---","metadata":{}},{"cell_type":"code","source":"\nlogger.info(\"Training regime classifier with balance checking using PCA features...\")\nprint(\"\\n🌲 Training regime classifier with balance checking using PCA features...\")\n\n# Load PCA dataset again for classifier training (we deleted it earlier for memory)\nprint(\"   📂 Reloading PCA dataset for classifier training...\")\nX_train_pca_classifier = pd.read_csv(pca_train_file)\nlogger.info(f\"Reloaded PCA dataset for classifier: {X_train_pca_classifier.shape}\")\n\n# Use PCA features (without target) for classifier training\nX_RegimeTrain = X_train_pca_classifier[pca_features_for_hmm].values\ny_RegimeTrain = X_train['predicted_regime']  # Use HMM predicted regimes as labels\n\nprint(f\"   🎯 Using {len(pca_features_for_hmm)} PCA components for classifier training\")\nlogger.info(f\"Classifier training with {len(pca_features_for_hmm)} PCA features\")\n\n# Standardize the PCA features for classifier\nscaler_classifier = StandardScaler()\nX_RegimeTrain_scaled = scaler_classifier.fit_transform(X_RegimeTrain)\n\n# Fit classifier with balance checking\nclf, best_auc, best_balance = fit_classifier_with_balance_check(X_RegimeTrain_scaled, y_RegimeTrain)\n\nprint(f\"✅ Final classifier - ROC AUC: {best_auc:.4f}, Balance ratio: {best_balance:.4f}\")\nlogger.info(f\"Final classifier performance: ROC AUC={best_auc:.4f}, Balance ratio={best_balance:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 5.00: Predict Regimes for Train and Test Data ---","metadata":{}},{"cell_type":"code","source":"logger.info(\"Predicting regimes for train and test data using PCA features...\")\nprint(\"🔮 Predicting regimes for train and test data using PCA features...\")\n\n# For training data (using classifier with PCA features)\nX_train_pca_scaled = scaler_classifier.transform(X_train_pca_classifier[pca_features_for_hmm])\nX_train['predicted_regime_Est'] = clf.predict(X_train_pca_scaled)\n\n# For test data (load the PCA test dataset)\nprint(\"   📂 Loading PCA test dataset for regime prediction...\")\nX_test_pca = pd.read_csv(pca_test_file)\nlogger.info(f\"Loaded PCA test dataset for prediction: {X_test_pca.shape}\")\n\n# Predict regimes for test data using PCA features\nX_test_pca_scaled = scaler_classifier.transform(X_test_pca[pca_features_for_hmm])\nX_test_pca['predicted_regime_Est'] = clf.predict(X_test_pca_scaled)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 5.25: Analyze Regime Properties ---","metadata":{}},{"cell_type":"code","source":"logger.info(\"Analyzing final regime distributions...\")\nprint(\"\\n📊 Final Regime Analysis:\")\n\nprint(\"\\nTrain Data Original HMM Regime Distribution:\")\norig_dist = X_train['predicted_regime'].value_counts()\nprint(orig_dist)\nlogger.info(f\"Train original regime distribution: {orig_dist.to_dict()}\")\n\nprint(\"\\nTrain Data Classifier Regime Distribution:\")\ntrain_dist = X_train['predicted_regime_Est'].value_counts()\nprint(train_dist)\nlogger.info(f\"Train classifier regime distribution: {train_dist.to_dict()}\")\n\nprint(\"\\nTest Data Regime Distribution:\")\ntest_dist = X_test_pca['predicted_regime_Est'].value_counts()\nprint(test_dist)\nlogger.info(f\"Test regime distribution: {test_dist.to_dict()}\")\n\n# Create regime dataframes for output\nlogger.info(\"Creating output regime dataframes...\")\nprint(\"💾 Creating output files...\")\n\ntrain_regimes_df = X_train[['timestamp', 'predicted_regime', 'predicted_regime_Est']].copy()\ntest_regimes_df = X_test_pca[['ID', 'predicted_regime_Est']].copy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* # --- Step 5.75: Output Regime DataFrames ---","metadata":{}},{"cell_type":"code","source":"# Save to CSV files\ntrain_regimes_df.to_csv('train_regimes.csv', index=False)\ntest_regimes_df.to_csv('test_regimes.csv', index=False)\n\nprint(\"\\n✅ Output saved to train_regimes.csv and test_regimes.csv\")\nprint(f\"\\n📊 Final Dataset Summary:\")\nprint(f\"   🔧 PCA Train Dataset: {pca_train_file}\")\nprint(f\"   🔧 PCA Test Dataset: {pca_test_file}\")\nprint(f\"   🎯 Top 500 Train Dataset: {top500_train_file}\")\nprint(f\"   🎯 Top 500 Test Dataset: {top500_test_file}\")\nprint(f\"   📋 Regime Predictions: train_regimes.csv, test_regimes.csv\")\nprint(f\"   🎯 Classifier uses: {len(pca_features_for_hmm)} PCA components (same as HMM)\")\nlogger.info(\"Regime detection pipeline completed successfully with PCA features\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}