{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"##### IMPORTS #####\n\"\"\"\nThis section imports all required libraries.\nCPU-based models are selected for performance optimization.\n\"\"\"\nimport os\nimport logging\nimport random\nimport time\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport librosa\nimport joblib\nimport cv2\nfrom pathlib import Path\nimport glob\n\n# Scikit-learn imports\nfrom sklearn.model_selection import train_test_split, GridSearchCV, validation_curve, learning_curve\nfrom sklearn.metrics import (confusion_matrix, classification_report, accuracy_score, \n                           precision_recall_fscore_support, make_scorer)\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_selection import SelectKBest, f_classif, chi2\n\n# CPU Optimized Classifiers\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.svm import SVC\n\n# Neural Network imports\ntry:\n    import tensorflow as tf\n    from tensorflow import keras\n    from tensorflow.keras import layers, Model\n    from tensorflow.keras.utils import to_categorical\n    TENSORFLOW_AVAILABLE = True\n    print(\"✓ TensorFlow loaded successfully\")\n    \n    # GPU Optimization Configuration\n    print(\"🔧 Configuring GPU optimizations...\")\n    \n    # Check GPU availability\n    gpus = tf.config.experimental.list_physical_devices('GPU')\n    if gpus:\n        try:\n            # Enable memory growth to prevent allocation of all GPU memory\n            for gpu in gpus:\n                tf.config.experimental.set_memory_growth(gpu, True)\n            \n            # Set mixed precision for faster training\n            policy = tf.keras.mixed_precision.Policy('mixed_float16')\n            tf.keras.mixed_precision.set_global_policy(policy)\n            \n            print(f\"✓ GPU Configuration Complete:\")\n            print(f\"   • Available GPUs: {len(gpus)}\")\n            print(f\"   • Memory growth enabled\")\n            print(f\"   • Mixed precision enabled (float16)\")\n            print(f\"   • GPU devices: {[gpu.name for gpu in gpus]}\")\n            \n        except RuntimeError as e:\n            print(f\"⚠️ GPU configuration error: {e}\")\n    else:\n        print(\"⚠️ No GPU detected, using CPU\")\n    \n    # Enable XLA compilation for faster execution\n    tf.config.optimizer.set_jit(True)\n    print(\"✓ XLA compilation enabled\")\n    \nexcept ImportError:\n    TENSORFLOW_AVAILABLE = False\n    print(\"⚠️ TensorFlow not available. Neural networks will be skipped.\")\n\nfrom tqdm.auto import tqdm\n\n# Performance optimizations\nwarnings.filterwarnings('ignore')\nwarnings.filterwarnings('ignore', category=UserWarning, module='sklearn')\nplt.style.use('default')  # Using default instead of seaborn for compatibility\n\n##### CONFIGURATION #####\n\"\"\"\nEnhanced configuration for better model performance with all new features\n\"\"\"\nclass CFG:\n    # Seed for reproducibility\n    seed = 42\n    debug = False  # Changed from True to False to use all data\n    \n    # Data paths (Kaggle paths)\n    train_datadir = '/kaggle/input/birdclef-2025/train_audio'\n    train_csv = '/kaggle/input/birdclef-2025/train.csv'\n    taxonomy_csv = '/kaggle/input/birdclef-2025/taxonomy.csv'\n    train_soundscapes_dir = '/kaggle/input/birdclef-2025/train_soundscapes'\n    test_soundscapes_dir = '/kaggle/input/birdclef-2025/test_soundscapes'\n    \n    # Audio processing parameters (optimized for speed)\n    FS = 16000  # Reduced from 32000 for faster processing\n    TARGET_DURATION = 5.0  \n    \n    # Mel spectrogram parameters (improved dimensions)\n    N_FFT = 1024  # Increased for better frequency resolution\n    HOP_LENGTH = 512  # Increased for better time resolution\n    N_MELS = 128  # More mel bands for better feature representation\n    FMIN = 50\n    FMAX = 8000  # Adjusted based on FS/2\n    TARGET_SHAPE = (128, 128)  # Larger dimensions for better features\n    \n    # Data Augmentation Settings (easy toggle)\n    enable_data_augmentation = True\n    augmentation_probability = 0.3  # 30% chance to apply each augmentation\n    \n    # Augmentation parameters\n    noise_factor = 0.02  # Background noise intensity\n    volume_range = (0.7, 1.3)  # Volume scaling range\n    mixup_alpha = 0.2  # Mixup parameter\n    \n    # Pseudo-labeling settings (easy toggle)\n    enable_pseudo_labeling = True  # Changed from False to True to enable\n    pseudo_confidence_threshold = 0.8  # Minimum confidence for pseudo-labels\n    pseudo_max_samples_per_class = 50  # Limit pseudo-samples per class\n    \n    # Quality filtering\n    filter_low_quality = True  # Remove samples with rating 0.5-2.5\n    min_quality_rating = 2.5  # Minimum rating to keep\n    \n    # Performance parameters\n    n_samples = None  # Use all data (was: 1500 if debug else None)\n    min_samples_for_rare_class_elimination = 10  # Higher threshold\n    test_size = 0.2\n    cv_folds = 3  # Keep at 3 for speed\n    \n    # PCA parameters\n    pca_variance_threshold = 0.95\n    \n    # GPU and Performance Optimization Parameters\n    use_multiprocessing = False  # Disabled for stability (parallel feature extraction was causing timeouts)\n    n_jobs = 1  # Use single thread for feature extraction\n    batch_size_neural_networks = 64  # Increased batch size for GPU\n    neural_network_epochs = 50  # Reduced epochs for faster training\n    early_stopping_patience = 5  # Reduced patience for faster convergence\n    enable_mixed_precision = True  # Use mixed precision training\n    enable_xla = True  # Enable XLA compilation\n    prefetch_buffer_size = 2  # Will be set to tf.data.AUTOTUNE if TensorFlow is available\n    \n    # Overfitting detection parameters\n    overfitting_threshold = 0.15  # Maximum acceptable gap between train and val loss\n    max_overfitting_models = 3  # Maximum number of overfitting models to skip before stopping\n    \n    # Memory optimization\n    clear_memory_between_models = True  # Clear memory between model trainings\n    reduce_model_architectures = debug  # Use fewer architectures in debug mode\n    \n    # Disable multiprocessing in debug mode for stability (already disabled by default for stability)\n    if debug:\n        use_multiprocessing = False\n        n_jobs = 1\n        print(\"🔬 Debug mode: Single-threaded processing for stability\")\n    \n    # Enhanced model configuration with additional models and better hyperparameters\n    models_to_train = {\n        'LogisticRegression': {\n            'model': LogisticRegression(max_iter=2000, solver='lbfgs', random_state=seed, n_jobs=-1),\n            'param_grid': {\n                'classifier__C': [0.01, 0.1, 1.0, 10.0],\n                'classifier__solver': ['lbfgs', 'liblinear']\n            }\n        },\n        'RandomForest': {\n            'model': RandomForestClassifier(random_state=seed, n_jobs=-1),\n            'param_grid': {\n                'classifier__n_estimators': [100, 200, 300],\n                'classifier__max_depth': [10, 20, None],\n                'classifier__min_samples_split': [2, 5],\n                'classifier__min_samples_leaf': [1, 2]\n            }\n        },\n        'DecisionTree': {\n            'model': DecisionTreeClassifier(random_state=seed),\n            'param_grid': {\n                'classifier__max_depth': [10, 20, 30, None],\n                'classifier__min_samples_split': [2, 5, 10],\n                'classifier__min_samples_leaf': [1, 2, 4],\n                'classifier__criterion': ['gini', 'entropy']\n            }\n        },\n        'KNeighbors': {\n            'model': KNeighborsClassifier(n_jobs=-1),\n            'param_grid': {\n                'classifier__n_neighbors': [3, 5, 7, 9],\n                'classifier__weights': ['uniform', 'distance'],\n                'classifier__metric': ['euclidean', 'manhattan']\n            }\n        },\n        'GaussianNB': {\n            'model': GaussianNB(),\n            'param_grid': {}\n        }\n    }\n\n##### UTILITY FUNCTIONS #####\n\"\"\"\nHelper functions and setup\n\"\"\"\ndef set_seed(seed=42):\n    \"\"\"Set seed for reproducibility\"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    print(f\"✓ Seed set: {seed}\")\n\ndef setup_logging():\n    \"\"\"Logging setup\"\"\"\n    logging.basicConfig(\n        level=logging.INFO,\n        format='%(asctime)s - %(levelname)s - %(message)s',\n        handlers=[\n            logging.StreamHandler(),\n            logging.FileHandler('training_log.log')\n        ]\n    )\n    print(\"=\"*60)\n    print(\"🚀 BirdCLEF Model Training Pipeline Started\")\n    print(\"=\"*60)\n\ndef print_section(title):\n    \"\"\"Helper function for printing section titles\"\"\"\n    print(\"\\n\" + \"=\"*60)\n    print(f\"📊 {title}\")\n    print(\"=\"*60)\n\ndef print_configuration(cfg):\n    \"\"\"Print configuration settings\"\"\"\n    print_section(\"⚙️ CONFIGURATION USED\")\n    \n    config_text = f\"\"\"\n⚙️ Configuration Used:\n• Data Augmentation: {'✓ Enabled' if cfg.enable_data_augmentation else '✗ Disabled'}\n• Pseudo-labeling: {'✓ Enabled' if cfg.enable_pseudo_labeling else '✗ Disabled'}\n• Quality Filtering: {'✓ Enabled' if cfg.filter_low_quality else '✗ Disabled'}\n\n📊 Data Parameters:\n• Sample Rate: {cfg.FS} Hz\n• Target Duration: {cfg.TARGET_DURATION}s\n• N_FFT: {cfg.N_FFT}\n• Hop Length: {cfg.HOP_LENGTH}\n• N_Mels: {cfg.N_MELS}\n• Target Shape: {cfg.TARGET_SHAPE}\n\n🧠 Model Parameters:\n• PCA Variance Threshold: {cfg.pca_variance_threshold*100}%\n• Test Size: {cfg.test_size*100}%\n• CV Folds: {cfg.cv_folds}\n• Debug Mode: {'✓ Enabled' if cfg.debug else '✗ Disabled'}\n• Sample Limit: {cfg.n_samples if cfg.debug else 'None (All data)'}\n\n⚡ GPU & Performance Optimizations:\n• Multiprocessing: {'✓ Enabled' if getattr(cfg, 'use_multiprocessing', False) else '✗ Disabled'}\n• N_Jobs: {getattr(cfg, 'n_jobs', 1)}\n• Neural Network Batch Size: {getattr(cfg, 'batch_size_neural_networks', 32)}\n• Neural Network Epochs: {getattr(cfg, 'neural_network_epochs', 100)}\n• Early Stopping Patience: {getattr(cfg, 'early_stopping_patience', 10)}\n• Mixed Precision: {'✓ Enabled' if getattr(cfg, 'enable_mixed_precision', False) else '✗ Disabled'}\n• XLA Compilation: {'✓ Enabled' if getattr(cfg, 'enable_xla', False) else '✗ Disabled'}\n• Memory Cleanup: {'✓ Enabled' if getattr(cfg, 'clear_memory_between_models', False) else '✗ Disabled'}\n• Reduced Architectures (Debug): {'✓ Enabled' if getattr(cfg, 'reduce_model_architectures', False) else '✗ Disabled'}\n\n🤖 Models to Train: {len(cfg.models_to_train)}\n• {', '.join(cfg.models_to_train.keys())}\n    \"\"\"\n    \n    print(config_text)\n\n##### DATA AUGMENTATION FUNCTIONS #####\n\"\"\"\nAudio data augmentation techniques\n\"\"\"\ndef add_background_noise(audio, noise_factor=0.02):\n    \"\"\"Add Gaussian background noise\"\"\"\n    noise = np.random.normal(0, noise_factor, len(audio))\n    return audio + noise\n\ndef volume_scaling(audio, volume_range=(0.7, 1.3)):\n    \"\"\"Apply random volume scaling\"\"\"\n    factor = np.random.uniform(volume_range[0], volume_range[1])\n    return audio * factor\n\ndef mixup_audio(audio1, audio2, alpha=0.2):\n    \"\"\"Apply mixup augmentation between two audio samples\"\"\"\n    lam = np.random.beta(alpha, alpha)\n    \n    # Ensure both audio samples have the same length\n    min_len = min(len(audio1), len(audio2))\n    audio1 = audio1[:min_len]\n    audio2 = audio2[:min_len]\n    \n    mixed_audio = lam * audio1 + (1 - lam) * audio2\n    return mixed_audio, lam\n\ndef apply_augmentation(audio, cfg, aug_type=None):\n    \"\"\"Apply random augmentation to audio\"\"\"\n    if not cfg.enable_data_augmentation:\n        return audio\n    \n    # Apply augmentation with probability\n    if np.random.random() > cfg.augmentation_probability:\n        return audio\n    \n    augmented_audio = audio.copy()\n    \n    # Background noise\n    if aug_type is None or aug_type == 'noise':\n        if np.random.random() < 0.5:\n            augmented_audio = add_background_noise(augmented_audio, cfg.noise_factor)\n    \n    # Volume scaling\n    if aug_type is None or aug_type == 'volume':\n        if np.random.random() < 0.5:\n            augmented_audio = volume_scaling(augmented_audio, cfg.volume_range)\n    \n    return augmented_audio\n\n##### ENHANCED AUDIO PROCESSING #####\n\"\"\"\nEnhanced audio processing functions with improved feature extraction\n\"\"\"\ndef extract_enhanced_audio_features(audio_path, cfg, apply_augmentation_flag=True):\n    \"\"\"\n    Enhanced audio feature extraction with multiple feature types:\n    - Mel spectrogram (primary features)\n    - MFCC features (cepstral coefficients)\n    - Spectral features (centroid, rolloff, zero crossing rate)\n    - Rhythm features (tempo, beat density)\n    \"\"\"\n    try:\n        y, sr = librosa.load(audio_path, sr=cfg.FS, duration=cfg.TARGET_DURATION)\n        \n        if len(y) == 0:\n            # Calculate correct feature dimension\n            base_size = cfg.TARGET_SHAPE[0] * cfg.TARGET_SHAPE[1]\n            additional_features = 13*4 + 6 + 2  # MFCC stats + spectral + rhythm\n            return np.zeros(base_size + additional_features, dtype=np.float32)\n        \n        # Normalize audio\n        y = librosa.util.normalize(y)\n        \n        # Apply data augmentation if enabled\n        if apply_augmentation_flag and cfg.enable_data_augmentation:\n            y = apply_augmentation(y, cfg)\n        \n        # 1. Mel spectrogram (primary features)\n        melspec = librosa.feature.melspectrogram(\n            y=y, sr=sr, n_fft=cfg.N_FFT, hop_length=cfg.HOP_LENGTH,\n            n_mels=cfg.N_MELS, fmin=cfg.FMIN, fmax=cfg.FMAX\n        )\n        melspec = librosa.power_to_db(melspec, ref=np.max)\n        melspec = (melspec - melspec.min()) / (melspec.max() - melspec.min() + 1e-8)\n        melspec = cv2.resize(melspec, (cfg.TARGET_SHAPE[1], cfg.TARGET_SHAPE[0]), \n                           interpolation=cv2.INTER_AREA)\n        \n        # 2. MFCC features\n        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13)\n        mfcc_stats = np.array([\n            np.mean(mfcc, axis=1),\n            np.std(mfcc, axis=1),\n            np.max(mfcc, axis=1),\n            np.min(mfcc, axis=1)\n        ]).flatten()\n        \n        # 3. Spectral features\n        spectral_centroid = librosa.feature.spectral_centroid(y=y, sr=sr)[0]\n        spectral_rolloff = librosa.feature.spectral_rolloff(y=y, sr=sr)[0]\n        zero_crossing_rate = librosa.feature.zero_crossing_rate(y)[0]\n        \n        # Aggregate spectral features\n        spectral_features = np.array([\n            np.mean(spectral_centroid), np.std(spectral_centroid),\n            np.mean(spectral_rolloff), np.std(spectral_rolloff),\n            np.mean(zero_crossing_rate), np.std(zero_crossing_rate)\n        ])\n        \n        # 4. Rhythm features\n        try:\n            tempo, beats = librosa.beat.beat_track(y=y, sr=sr)\n            rhythm_features = np.array([tempo, len(beats) / len(y) * sr])  # tempo and beat density\n        except:\n            rhythm_features = np.array([0.0, 0.0])  # fallback\n        \n        # Combine all features\n        melspec_features = melspec.flatten()\n        combined_features = np.concatenate([\n            melspec_features,\n            mfcc_stats,\n            spectral_features,\n            rhythm_features\n        ])\n        \n        return combined_features.astype(np.float32)\n        \n    except Exception as e:\n        print(f\"⚠️ Audio processing error for {audio_path}: {e}\")\n        # Return zeros with correct dimension\n        base_size = cfg.TARGET_SHAPE[0] * cfg.TARGET_SHAPE[1]\n        additional_features = 13*4 + 6 + 2  # MFCC stats + spectral + rhythm\n        return np.zeros(base_size + additional_features, dtype=np.float32)\n\ndef audio_to_melspec(audio_path, cfg):\n    \"\"\"Legacy mel spectrogram extraction for backward compatibility\"\"\"\n    return extract_enhanced_audio_features(audio_path, cfg)\n\n##### PSEUDO-LABELING FUNCTIONS #####\n\"\"\"\nPseudo-labeling implementation for soundscapes\n\"\"\"\ndef extract_soundscape_files(cfg):\n    \"\"\"Extract .ogg files from train_soundscapes directory\"\"\"\n    if not os.path.exists(cfg.train_soundscapes_dir):\n        print(f\"⚠️ Soundscapes directory not found: {cfg.train_soundscapes_dir}\")\n        return []\n    \n    # Find all .ogg files\n    ogg_files = glob.glob(os.path.join(cfg.train_soundscapes_dir, \"**/*.ogg\"), recursive=True)\n    print(f\"✓ Found {len(ogg_files)} soundscape files\")\n    return ogg_files\n\ndef generate_pseudo_labels(model, soundscape_files, cfg, label_encoder):\n    \"\"\"Generate pseudo-labels for soundscape files\"\"\"\n    if not cfg.enable_pseudo_labeling or not soundscape_files:\n        return [], []\n    \n    print_section(\"PSEUDO-LABELING\")\n    print(f\"🏷️ Generating pseudo-labels for {len(soundscape_files)} files...\")\n    \n    pseudo_features = []\n    pseudo_labels = []\n    \n    for file_path in tqdm(soundscape_files[:200], desc=\"Processing soundscapes\"):  # Limit for speed\n        try:\n            # Extract features (without augmentation for inference)\n            features = extract_enhanced_audio_features(file_path, cfg, apply_augmentation_flag=False)\n            \n            if np.all(features == 0):\n                continue\n            \n            # Predict with confidence\n            features_reshaped = features.reshape(1, -1)\n            if hasattr(model, 'predict_proba'):\n                probabilities = model.predict_proba(features_reshaped)[0]\n                confidence = np.max(probabilities)\n                predicted_class = np.argmax(probabilities)\n            else:\n                # For models without predict_proba, use decision function if available\n                predicted_class = model.predict(features_reshaped)[0]\n                confidence = 0.5  # Default moderate confidence\n            \n            # Only keep high-confidence predictions\n            if confidence >= cfg.pseudo_confidence_threshold:\n                pseudo_features.append(features)\n                pseudo_labels.append(predicted_class)\n                \n        except Exception as e:\n            print(f\"⚠️ Error processing {file_path}: {e}\")\n            continue\n    \n    # Limit samples per class\n    if pseudo_features:\n        pseudo_features = np.array(pseudo_features)\n        pseudo_labels = np.array(pseudo_labels)\n        \n        # Balance pseudo-samples per class\n        balanced_features = []\n        balanced_labels = []\n        \n        for class_id in np.unique(pseudo_labels):\n            class_mask = pseudo_labels == class_id\n            class_features = pseudo_features[class_mask]\n            \n            # Limit samples per class\n            if len(class_features) > cfg.pseudo_max_samples_per_class:\n                indices = np.random.choice(len(class_features), cfg.pseudo_max_samples_per_class, replace=False)\n                class_features = class_features[indices]\n            \n            balanced_features.extend(class_features)\n            balanced_labels.extend([class_id] * len(class_features))\n        \n        pseudo_features = np.array(balanced_features)\n        pseudo_labels = np.array(balanced_labels)\n        \n        print(f\"✓ Generated {len(pseudo_features)} pseudo-labeled samples\")\n        \n        # Show distribution\n        unique, counts = np.unique(pseudo_labels, return_counts=True)\n        for class_id, count in zip(unique, counts):\n            class_name = label_encoder.classes_[class_id] if class_id < len(label_encoder.classes_) else \"Unknown\"\n            print(f\"   {class_name}: {count} samples\")\n    \n    return pseudo_features, pseudo_labels\n\n##### DATA PREPARATION #####\n\"\"\"\nVeri hazırlama ve ön işleme\n\"\"\"\n\ndef process_audio_chunk(chunk_data):\n    \"\"\"Process a chunk of audio files for parallel processing\"\"\"\n    chunk_df, cfg = chunk_data\n    chunk_features = []\n    chunk_labels = []\n    chunk_aug_count = 0\n    \n    for _, row in chunk_df.iterrows():\n        try:\n            feature_vector = extract_enhanced_audio_features(row['filepath'], cfg)\n            \n            if not np.all(feature_vector == 0):\n                chunk_features.append(feature_vector)\n                chunk_labels.append(row['target'])\n                \n                if cfg.enable_data_augmentation and np.random.random() < cfg.augmentation_probability:\n                    chunk_aug_count += 1\n                    \n        except Exception as e:\n            print(f\"⚠️ Skipping {row['filepath']}: {e}\")\n            continue\n    \n    return chunk_features, chunk_labels, chunk_aug_count\n\ndef prepare_data(cfg):\n    \"\"\"Enhanced data preparation with metadata utilization\"\"\"\n    print_section(\"DATA PREPARATION\")\n    \n    # Create dummy data for testing if files don't exist\n    if not os.path.exists(cfg.train_csv):\n        print(\"⚠️ Data files not found. Creating demo data...\")\n        return create_dummy_data(cfg)\n    \n    # Load data\n    print(\"📂 Loading data files...\")\n    train_df = pd.read_csv(cfg.train_csv)\n    \n    # Load taxonomy data if available\n    if os.path.exists(cfg.taxonomy_csv):\n        taxonomy_df = pd.read_csv(cfg.taxonomy_csv)\n        print(f\"✓ Loaded taxonomy data: {len(taxonomy_df)} species\")\n        \n        # Merge with train data to get additional species information\n        train_df = train_df.merge(taxonomy_df, left_on='primary_label', right_on='primary_label', how='left')\n    \n    print(f\"✓ {len(train_df)} samples loaded\")\n    \n    # Debug mode sampling\n    if cfg.debug and cfg.n_samples and cfg.n_samples < len(train_df):\n        train_df = train_df.sample(cfg.n_samples, random_state=cfg.seed)\n        print(f\"🔬 Debug mode: {cfg.n_samples} samples selected\")\n    \n    # Enhanced data filtering\n    print(\"🧹 Cleaning data...\")\n    \n    # Remove samples with missing primary labels\n    initial_len = len(train_df)\n    train_df = train_df.dropna(subset=['primary_label'])\n    if len(train_df) < initial_len:\n        print(f\"✓ Removed {initial_len - len(train_df)} samples with missing labels\")\n    \n    # Filter by quality rating (remove low quality samples)\n    if 'rating' in train_df.columns and cfg.filter_low_quality:\n        initial_len = len(train_df)\n        # Remove samples with rating between 0.5 and 2.5\n        quality_filtered = train_df[~((train_df['rating'] >= 0.5) & (train_df['rating'] <= cfg.min_quality_rating))]\n        train_df = quality_filtered\n        print(f\"✓ Filtered low quality samples (rating 0.5-{cfg.min_quality_rating}): removed {initial_len - len(train_df)} samples\")\n    \n    # Remove rare classes\n    class_counts = train_df['primary_label'].value_counts()\n    rare_classes = class_counts[class_counts < cfg.min_samples_for_rare_class_elimination].index\n    \n    if len(rare_classes) > 0:\n        initial_len = len(train_df)\n        train_df = train_df[~train_df['primary_label'].isin(rare_classes)]\n        print(f\"✓ {len(rare_classes)} rare classes eliminated ({initial_len - len(train_df)} samples)\")\n    \n    # Create file paths\n    train_df['filepath'] = train_df['filename'].apply(lambda x: os.path.join(cfg.train_datadir, x))\n    \n    # Encode labels\n    le = LabelEncoder()\n    train_df['target'] = le.fit_transform(train_df['primary_label'])\n    \n    # Save label encoder\n    joblib.dump(le, \"label_encoder.joblib\")\n    \n    # Update config\n    cfg.num_classes = len(le.classes_)\n    cfg.class_names = le.classes_\n    \n    print(f\"✓ {cfg.num_classes} classes, {len(train_df)} samples ready\")\n    \n    # Enhanced class distribution analysis\n    class_dist = train_df['primary_label'].value_counts()\n    print(f\"📊 Top 10 most common classes: {dict(class_dist.head(10))}\")\n    print(f\"📊 Class distribution stats: min={class_dist.min()}, max={class_dist.max()}, mean={class_dist.mean():.1f}\")\n    \n    return train_df, le\n\ndef create_dummy_data(cfg):\n    \"\"\"Create dummy data for demo purposes\"\"\"\n    print(\"🎭 Creating demo data...\")\n    \n    # Create dummy audio files and dataframe\n    n_samples = 500  # Increased for better testing\n    bird_species = ['robin', 'sparrow', 'eagle', 'hawk', 'crow', 'owl', 'cardinal', 'bluejay', 'woodpecker', 'finch']\n    \n    data = []\n    for i in range(n_samples):\n        species = np.random.choice(bird_species)\n        filename = f\"{species}_{i:03d}.wav\"\n        # Add some metadata with realistic rating distribution\n        rating = np.random.choice([0, 1, 2, 3, 4, 5], p=[0.1, 0.1, 0.2, 0.3, 0.2, 0.1])\n        data.append({\n            'filename': filename,\n            'primary_label': species,\n            'rating': rating,  # Quality rating 0-5\n            'latitude': np.random.uniform(-90, 90),\n            'longitude': np.random.uniform(-180, 180),\n            'author': f\"user_{np.random.randint(1, 50)}\"\n        })\n    \n    train_df = pd.DataFrame(data)\n    train_df['filepath'] = train_df['filename']  # Use dummy paths\n    \n    # Encode labels\n    le = LabelEncoder()\n    train_df['target'] = le.fit_transform(train_df['primary_label'])\n    \n    # Save label encoder\n    joblib.dump(le, \"label_encoder.joblib\")\n    \n    # Update config\n    cfg.num_classes = len(le.classes_)\n    cfg.class_names = le.classes_\n    \n    print(f\"✓ Demo data ready: {cfg.num_classes} classes, {len(train_df)} samples\")\n    \n    return train_df, le\n\ndef extract_features(df, cfg):\n    \"\"\"Enhanced feature extraction with progress tracking and augmentation - Sequential Processing Only\"\"\"\n    print_section(\"FEATURE EXTRACTION\")\n    \n    print(\"🎵 Extracting enhanced audio features...\")\n    \n    # Display data augmentation configuration\n    if cfg.enable_data_augmentation:\n        print(\"\\n🔄 Data Augmentation Optimizations Applied:\")\n        print(f\"   • Background Noise: ✓ Enabled (factor: {cfg.noise_factor})\")\n        print(f\"   • Volume Scaling: ✓ Enabled (range: {cfg.volume_range})\")\n        print(f\"   • Mixup Alpha: {cfg.mixup_alpha}\")\n        print(f\"   • Augmentation Probability: {cfg.augmentation_probability * 100}%\")\n        print(\"   • Multi-modal Features: Mel + MFCC + Spectral + Rhythm\")\n    else:\n        print(\"\\n🚫 Data Augmentation: Disabled\")\n    \n    # Performance settings\n    print(f\"⚡ Processing Method: Sequential (reliable and stable)\")\n    print(f\"   • Processing mode: Single-threaded for maximum stability\")\n    print(f\"   • Memory optimization: Enabled\")\n    \n    # Detailed Feature Breakdown\n    base_size = cfg.TARGET_SHAPE[0] * cfg.TARGET_SHAPE[1]\n    additional_features = 13*4 + 6 + 2  # MFCC stats + spectral + rhythm\n    n_features = base_size + additional_features\n    \n    print(f\"\\n🎵 Multi-Modal Feature Breakdown:\")\n    print(f\"   📊 Mel Spectrogram (Primary Features):\")\n    print(f\"      → {base_size} features ({cfg.TARGET_SHAPE})\")\n    print(f\"      → Frequency content over time (what frequencies are present when)\")\n    print(f\"      → Captures bird call patterns, harmonics, and spectral envelope\")\n    print(f\"   🎼 MFCC Features (Cepstral Analysis):\")\n    print(f\"      → {13*4} features (13 coeffs × 4 statistics: mean, std, max, min)\")\n    print(f\"      → Captures vocal tract shape and timbre characteristics\")\n    print(f\"      → Essential for species-specific vocal signature recognition\")\n    print(f\"   🌊 Spectral Features (Signal Properties):\")\n    print(f\"      → 6 features (centroid, rolloff, zero-crossing rate)\")\n    print(f\"      → Spectral Centroid: brightness/sharpness of sound\")\n    print(f\"      → Spectral Rolloff: energy distribution across frequencies\")\n    print(f\"      → Zero Crossing Rate: noisiness vs tonality\")\n    print(f\"   🥁 Rhythm Features (Temporal Patterns):\")\n    print(f\"      → 2 features (tempo, beat density)\")\n    print(f\"      → Tempo: rhythmic speed of bird calls\")\n    print(f\"      → Beat Density: timing patterns and call repetition rate\")\n    \n    print(f\"\\n📈 Combined Feature Power:\")\n    print(f\"   • Total Feature Dimension: {n_features}\")\n    print(f\"   • Mel Spectrogram: {base_size} ({base_size/n_features*100:.1f}%)\")\n    print(f\"   • MFCC Statistics: {13*4} ({13*4/n_features*100:.1f}%)\")\n    print(f\"   • Spectral Features: 6 ({6/n_features*100:.1f}%)\")\n    print(f\"   • Rhythm Features: 2 ({2/n_features*100:.1f}%)\")\n    print(f\"   → Comprehensive audio signature for species identification\")\n    \n    # For demo data, create more sophisticated random features\n    if not os.path.exists(cfg.train_datadir):\n        print(\"🎭 Creating enhanced demo features...\")\n        \n        # Use the already calculated feature dimensions\n        print(f\"🚀 Using optimized vectorized feature generation...\")\n        \n        # Vectorized feature generation for speed\n        n_samples = len(df)\n        y = df['target'].values\n        \n        # Pre-allocate arrays for better memory usage\n        X = np.zeros((n_samples, n_features), dtype=np.float32)\n        \n        # Vectorized generation using broadcasting\n        species_ids = y.reshape(-1, 1)\n        \n        # Base mel spectrogram features\n        mel_base = np.random.rand(n_samples, base_size).astype(np.float32) * 0.5\n        mel_patterns = species_ids * 0.1  # Species-specific patterns\n        X[:, :base_size] = mel_base + mel_patterns\n        \n        # MFCC features\n        mfcc_base = np.random.rand(n_samples, 52).astype(np.float32) * 0.3\n        mfcc_patterns = species_ids * 0.05\n        X[:, base_size:base_size+52] = mfcc_base + mfcc_patterns\n        \n        # Spectral and rhythm features\n        remaining_features = n_features - base_size - 52\n        other_base = np.random.rand(n_samples, remaining_features).astype(np.float32) * 0.2\n        other_patterns = species_ids * 0.02\n        X[:, base_size+52:] = other_base + other_patterns\n        \n        print(f\"✓ Enhanced demo feature matrix: {X.shape} (vectorized generation)\")\n        return X, y\n    \n    # Sequential feature extraction - reliable and stable\n    features = []\n    labels = []\n    \n    start_time = time.time()\n    augmentation_count = 0\n    \n    print(\"🔄 Using sequential feature extraction (stable and reliable)...\")\n    for idx, row in tqdm(df.iterrows(), total=len(df), desc=\"Processing Audio\"):\n        try:\n            feature_vector = extract_enhanced_audio_features(row['filepath'], cfg)\n            \n            if not np.all(feature_vector == 0):\n                features.append(feature_vector)\n                labels.append(row['target'])\n                \n                # Count augmentations applied (rough estimate)\n                if cfg.enable_data_augmentation and np.random.random() < cfg.augmentation_probability:\n                    augmentation_count += 1\n                    \n        except Exception as e:\n            print(f\"⚠️ Skipping {row['filepath']}: {e}\")\n            continue\n    \n    X = np.array(features, dtype=np.float32)  # Use float32 for memory efficiency\n    y = np.array(labels)\n    \n    elapsed_time = time.time() - start_time\n    print(f\"✓ Feature extraction completed: {X.shape}, {elapsed_time:.2f} seconds\")\n    \n    # Safe division to avoid division by zero\n    if len(X) > 0:\n        print(f\"⚡ Processing speed: {len(X)/elapsed_time:.2f} samples/second\")\n    else:\n        print(\"⚡ Processing speed: 0.00 samples/second\")\n    \n    if cfg.enable_data_augmentation:\n        print(f\"🔄 Augmentation Summary:\")\n        print(f\"   • ~{augmentation_count} samples received augmentation\")\n        if len(X) > 0:\n            print(f\"   • Augmentation rate: ~{(augmentation_count/len(X)*100):.1f}%\")\n        else:\n            print(f\"   • Augmentation rate: 0.0%\")\n    \n    if X.shape[0] == 0:\n        raise ValueError(\"❌ No samples remaining after feature extraction!\")\n    \n    return X, y\n\ndef apply_mixup_features(X, y, cfg):\n    \"\"\"Apply mixup augmentation to feature vectors\"\"\"\n    if not cfg.enable_data_augmentation:\n        return X, y\n    \n    print(\"🔄 Applying mixup augmentation...\")\n    print(f\"   • Original samples: {len(X)}\")\n    \n    mixed_X = []\n    mixed_y = []\n    \n    # Keep original data\n    mixed_X.extend(X)\n    mixed_y.extend(y)\n    \n    # Generate mixup samples\n    n_mixup = int(len(X) * 0.2)  # 20% additional mixup samples\n    print(f\"   • Generating {n_mixup} mixup samples (20% of original)\")\n    print(f\"   • Mixup alpha parameter: {cfg.mixup_alpha}\")\n    \n    for i in range(n_mixup):\n        # Select two random samples\n        idx1, idx2 = np.random.choice(len(X), 2, replace=False)\n        \n        # Mixup features\n        lam = np.random.beta(cfg.mixup_alpha, cfg.mixup_alpha)\n        mixed_feature = lam * X[idx1] + (1 - lam) * X[idx2]\n        \n        # For classification, use the label of the dominant sample\n        mixed_label = y[idx1] if lam > 0.5 else y[idx2]\n        \n        mixed_X.append(mixed_feature)\n        mixed_y.append(mixed_label)\n    \n    X_mixed = np.array(mixed_X)\n    y_mixed = np.array(mixed_y)\n    \n    print(f\"✓ Mixup applied: {X.shape} → {X_mixed.shape}\")\n    print(f\"   • Data increase: {((len(X_mixed) - len(X)) / len(X) * 100):.1f}%\")\n    \n    return X_mixed, y_mixed\n\n##### PCA ANALYSIS #####\n\"\"\"\nPCA analizi ve optimum bileşen sayısı belirleme\n\"\"\"\ndef analyze_pca_components(X_train, cfg):\n    \"\"\"Determine optimal number of components through PCA analysis - optimized version\"\"\"\n    print_section(\"PCA ANALYSIS\")\n    \n    print(\"🔍 Analyzing explained variance with PCA...\")\n    \n    # Optimization: Use randomized SVD for faster computation on large datasets\n    n_samples, n_features = X_train.shape\n    use_randomized = n_features > 1000 or n_samples > 5000\n    \n    if use_randomized:\n        print(\"⚡ Using randomized SVD for faster computation on large dataset\")\n        # Estimate optimal n_components for randomized PCA\n        max_components = min(n_samples, n_features, 500)  # Limit for speed\n        pca_full = PCA(n_components=max_components, svd_solver='randomized', random_state=cfg.seed)\n    else:\n        print(\"🔄 Using full SVD for complete analysis\")\n        pca_full = PCA(n_components=None, random_state=cfg.seed)\n    \n    # Optimized fitting with progress\n    print(f\"   • Dataset shape: {X_train.shape}\")\n    print(f\"   • Analysis method: {'Randomized SVD' if use_randomized else 'Full SVD'}\")\n    \n    start_time = time.time()\n    pca_full.fit(X_train)\n    pca_time = time.time() - start_time\n    \n    print(f\"✓ PCA analysis completed in {pca_time:.2f} seconds\")\n    \n    cumulative_variance = np.cumsum(pca_full.explained_variance_ratio_)\n    \n    # Visualization with optimized plotting\n    plt.figure(figsize=(12, 8))\n    \n    plt.subplot(2, 2, 1)\n    plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, \n             marker='o', linestyle='--', markersize=3)\n    plt.xlabel('Number of Components')\n    plt.ylabel('Cumulative Explained Variance')\n    plt.title('Cumulative Explained Variance')\n    plt.grid(True, alpha=0.3)\n    plt.axhline(y=cfg.pca_variance_threshold, color='r', linestyle=':', \n                label=f'{cfg.pca_variance_threshold*100}% Variance Threshold')\n    plt.legend()\n    \n    # Individual variance contribution (limit to first 20 for clarity)\n    plt.subplot(2, 2, 2)\n    n_components_to_show = min(20, len(pca_full.explained_variance_ratio_))\n    plt.bar(range(1, n_components_to_show + 1), \n            pca_full.explained_variance_ratio_[:n_components_to_show])\n    plt.xlabel('Component Number')\n    plt.ylabel('Explained Variance Ratio')\n    plt.title(f'Variance Contribution of First {n_components_to_show} Components')\n    plt.grid(True, alpha=0.3)\n    \n    # Find optimal number of components\n    n_components_chosen = np.argmax(cumulative_variance >= cfg.pca_variance_threshold) + 1\n    \n    # Ensure we don't exceed the number of available components\n    n_components_chosen = min(n_components_chosen, len(cumulative_variance))\n    \n    # Show different thresholds\n    plt.subplot(2, 2, 3)\n    thresholds = [0.90, 0.95, 0.99]\n    threshold_components = []\n    for thresh in thresholds:\n        # Handle case where threshold might not be reached\n        thresh_idx = np.argmax(cumulative_variance >= thresh)\n        if cumulative_variance[thresh_idx] >= thresh:\n            n_comp = thresh_idx + 1\n        else:\n            n_comp = len(cumulative_variance)  # Use all components\n        threshold_components.append(n_comp)\n        plt.axhline(y=thresh, linestyle='--', alpha=0.7, \n                   label=f'{thresh*100}% → {n_comp} components')\n    \n    plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, 'b-', alpha=0.7)\n    plt.xlabel('Number of Components')\n    plt.ylabel('Cumulative Variance')\n    plt.title('Different Variance Thresholds')\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    \n    # Summary table\n    plt.subplot(2, 2, 4)\n    plt.axis('off')\n    \n    # Calculate compression ratio\n    compression_ratio = (n_features - n_components_chosen) / n_features * 100\n    \n    summary_text = f\"\"\"PCA ANALYSIS RESULTS\n\nTotal feature count: {n_features:,}\nSamples analyzed: {n_samples:,}\nAnalysis time: {pca_time:.2f}s\n\nVariance Thresholds:\n• 90% variance: {threshold_components[0]} components\n• 95% variance: {threshold_components[1]} components  \n• 99% variance: {threshold_components[2]} components\n\nSelected: {n_components_chosen} components\n({cfg.pca_variance_threshold*100}% variance)\n\nDimensionality reduction: \n{n_features:,} → {n_components_chosen:,}\n({compression_ratio:.1f}% compression)\n\nSpeed: {n_samples*n_features/pca_time/1e6:.1f}M ops/sec\"\"\"\n    \n    plt.text(0.1, 0.5, summary_text, transform=plt.gca().transAxes, \n             fontsize=10, verticalalignment='center', fontfamily='monospace')\n    \n    plt.tight_layout()\n    plt.savefig('pca_analysis.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    print(\"✓ PCA analysis completed\")\n    print(f\"📊 {n_components_chosen} components selected for {cfg.pca_variance_threshold*100}% variance\")\n    print(f\"📉 Dimensionality reduction: {n_features:,} → {n_components_chosen:,} ({compression_ratio:.1f}% compression)\")\n    print(f\"⚡ Processing speed: {n_samples*n_features/pca_time/1e6:.1f}M operations/second\")\n    \n    return n_components_chosen\n\ndef apply_pca_transformation(X_train, X_test, n_components, cfg):\n    \"\"\"Apply PCA transformation\"\"\"\n    print(f\"🔄 Applying PCA transformation ({n_components} components)...\")\n    \n    pca = PCA(n_components=n_components, random_state=cfg.seed)\n    X_train_reduced = pca.fit_transform(X_train)\n    X_test_reduced = pca.transform(X_test)\n    \n    # Save PCA\n    joblib.dump(pca, \"pca_transformer.joblib\")\n    \n    print(f\"✓ PCA applied: {X_train.shape} → {X_train_reduced.shape}\")\n    \n    return X_train_reduced, X_test_reduced, pca\n\n##### MODEL TRAINING & VALIDATION #####\n\"\"\"\nModel eğitimi ve doğrulama\n\"\"\"\ndef train_and_evaluate_models(X_train, y_train, X_test, y_test, cfg, label_encoder):\n    \"\"\"Optimized model training with validation\"\"\"\n    print_section(\"MODEL TRAINING AND EVALUATION\")\n    \n    results = []\n    best_model = None\n    best_model_name = \"\"\n    best_accuracy = -1\n    \n    for model_name, model_info in cfg.models_to_train.items():\n        print(f\"\\n🤖 Training model: {model_name}\")\n        \n        # Create pipeline\n        pipeline = Pipeline([\n            ('scaler', StandardScaler()),\n            ('classifier', model_info['model'])\n        ])\n        \n        start_time = time.time()\n        \n        if not model_info['param_grid']:\n            # No hyperparameter tuning\n            print(\"⚡ Direct training (no hyperparameter optimization)\")\n            pipeline.fit(X_train, y_train)\n            best_estimator = pipeline\n            \n            # Cross-validation score\n            from sklearn.model_selection import cross_val_score\n            cv_scores = cross_val_score(pipeline, X_train, y_train, cv=cfg.cv_folds, \n                                      scoring='accuracy', n_jobs=-1)\n            cv_mean = cv_scores.mean()\n            cv_std = cv_scores.std()\n            \n        else:\n            # Grid search with cross-validation\n            print(\"🔍 Starting GridSearchCV...\")\n            grid_search = GridSearchCV(\n                pipeline, model_info['param_grid'],\n                cv=cfg.cv_folds, scoring='accuracy',\n                n_jobs=-1, verbose=1\n            )\n            \n            grid_search.fit(X_train, y_train)\n            best_estimator = grid_search.best_estimator_\n            cv_mean = grid_search.best_score_\n            cv_std = grid_search.cv_results_['std_test_score'][grid_search.best_index_]\n            \n            print(f\"✓ Best parameters: {grid_search.best_params_}\")\n        \n        # Test performance\n        test_accuracy = accuracy_score(y_test, best_estimator.predict(X_test))\n        training_time = time.time() - start_time\n        \n        # Store results\n        result = {\n            'model_name': model_name,\n            'cv_mean': cv_mean,\n            'cv_std': cv_std,\n            'test_accuracy': test_accuracy,\n            'training_time': training_time,\n            'best_estimator': best_estimator\n        }\n        results.append(result)\n        \n        # Update best model\n        if test_accuracy > best_accuracy:\n            best_accuracy = test_accuracy\n            best_model = best_estimator\n            best_model_name = model_name\n        \n        print(f\"📊 {model_name} Results:\")\n        print(f\"   CV Accuracy: {cv_mean:.4f} (±{cv_std:.4f})\")\n        print(f\"   Test Accuracy: {test_accuracy:.4f}\")\n        print(f\"   Training Time: {training_time:.2f} seconds\")\n    \n    # Results summary\n    print_section(\"MODEL COMPARISON RESULTS\")\n    \n    results_df = pd.DataFrame([{\n        'Model': r['model_name'],\n        'CV Accuracy': f\"{r['cv_mean']:.4f} ± {r['cv_std']:.4f}\",\n        'Test Accuracy': f\"{r['test_accuracy']:.4f}\",\n        'Training Time (s)': f\"{r['training_time']:.2f}\"\n    } for r in results])\n    \n    print(results_df.to_string(index=False))\n    \n    print(f\"\\n🏆 BEST MODEL: {best_model_name} (Test Accuracy: {best_accuracy:.4f})\")\n    \n    return best_model, best_model_name, results\n\n##### DETAILED EVALUATION #####\n\"\"\"\nDetaylı model değerlendirmesi\n\"\"\"\ndef detailed_model_evaluation(model, X_test, y_test, label_encoder, model_name):\n    \"\"\"Comprehensive model evaluation\"\"\"\n    print_section(f\"DETAILED EVALUATION - {model_name}\")\n    \n    # Predictions\n    y_pred = model.predict(X_test)\n    \n    # Metrics\n    accuracy = accuracy_score(y_test, y_pred)\n    precision, recall, f1, support = precision_recall_fscore_support(y_test, y_pred, average='weighted')\n    \n    print(f\"📊 {model_name} Test Metrics:\")\n    print(f\"   Accuracy: {accuracy:.4f}\")\n    print(f\"   Precision: {precision:.4f}\")\n    print(f\"   Recall: {recall:.4f}\")\n    print(f\"   F1-Score: {f1:.4f}\")\n    \n    # Classification Report\n    print(\"\\n📋 CLASSIFICATION REPORT:\")\n    print(\"-\" * 60)\n    class_report = classification_report(y_test, y_pred, target_names=label_encoder.classes_, \n                                       zero_division=0)\n    print(class_report)\n    \n    # Confusion Matrix Visualization\n    plt.figure(figsize=(15, 10))\n    \n    # Main confusion matrix\n    plt.subplot(2, 2, 1)\n    cm = confusion_matrix(y_test, y_pred)\n    cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n    \n    # Show only top classes for clarity\n    max_classes = min(15, len(label_encoder.classes_))\n    top_classes_idx = np.argsort(np.bincount(y_test))[-max_classes:]\n    \n    cm_display = cm_norm[top_classes_idx][:, top_classes_idx]\n    class_names_display = label_encoder.classes_[top_classes_idx]\n    \n    sns.heatmap(cm_display, annot=True, fmt='.2f', cmap='Blues',\n                xticklabels=class_names_display, yticklabels=class_names_display)\n    plt.title(f'Confusion Matrix - {model_name}\\n(Top {max_classes} Classes)')\n    plt.ylabel('Actual')\n    plt.xlabel('Predicted')\n    plt.xticks(rotation=45)\n    plt.yticks(rotation=0)\n    \n    # Per-class accuracy\n    plt.subplot(2, 2, 2)\n    per_class_acc = cm.diagonal() / cm.sum(axis=1)\n    class_counts = np.bincount(y_test)\n    \n    # Sort by accuracy\n    sorted_idx = np.argsort(per_class_acc)[-15:]  # Top 15\n    \n    plt.barh(range(len(sorted_idx)), per_class_acc[sorted_idx])\n    plt.yticks(range(len(sorted_idx)), label_encoder.classes_[sorted_idx])\n    plt.xlabel('Class Accuracy')\n    plt.title('Per-Class Accuracy (Top 15)')\n    plt.grid(True, alpha=0.3)\n    \n    # Class distribution in test set\n    plt.subplot(2, 2, 3)\n    test_class_counts = pd.Series(y_test).value_counts().head(15)\n    test_class_names = [label_encoder.classes_[i] for i in test_class_counts.index]\n    \n    plt.bar(range(len(test_class_counts)), test_class_counts.values)\n    plt.xticks(range(len(test_class_counts)), test_class_names, rotation=45)\n    plt.ylabel('Sample Count')\n    plt.title('Class Distribution in Test Set (Top 15)')\n    plt.grid(True, alpha=0.3)\n    \n    # Model performance summary\n    plt.subplot(2, 2, 4)\n    plt.axis('off')\n    summary_text = f\"\"\"MODEL PERFORMANCE SUMMARY\n\nModel: {model_name}\n\nOverall Metrics:\n• Accuracy: {accuracy:.4f}\n• Precision: {precision:.4f}\n• Recall: {recall:.4f}\n• F1-Score: {f1:.4f}\n\nTest Set:\n• Total samples: {len(y_test)}\n• Number of classes: {len(np.unique(y_test))}\n• Best class accuracy: {per_class_acc.max():.4f}\n• Worst class accuracy: {per_class_acc.min():.4f}\"\"\"\n    \n    plt.text(0.1, 0.5, summary_text, transform=plt.gca().transAxes,\n             fontsize=11, verticalalignment='center', fontfamily='monospace')\n    \n    plt.tight_layout()\n    plt.savefig(f'evaluation_{model_name.lower()}.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    return accuracy, precision, recall, f1\n\n##### FEATURE IMPORTANCE ANALYSIS #####\n\"\"\"\nÖzellik önem analizi\n\"\"\"\ndef analyze_feature_importance(model, model_name, pca, feature_names=None):\n    \"\"\"Feature importance analysis for supported models\"\"\"\n    print_section(f\"FEATURE IMPORTANCE ANALYSIS - {model_name}\")\n    \n    try:\n        # Get the actual classifier from pipeline\n        if hasattr(model, 'named_steps'):\n            classifier = model.named_steps['classifier']\n        else:\n            classifier = model\n        \n        importance_scores = None\n        importance_type = \"\"\n        \n        # Random Forest\n        if hasattr(classifier, 'feature_importances_'):\n            importance_scores = classifier.feature_importances_\n            importance_type = \"Gini Importance\"\n            \n        # Logistic Regression\n        elif hasattr(classifier, 'coef_'):\n            if len(classifier.coef_.shape) > 1:\n                # Multi-class: use mean absolute coefficients\n                importance_scores = np.mean(np.abs(classifier.coef_), axis=0)\n            else:\n                importance_scores = np.abs(classifier.coef_[0])\n            importance_type = \"Coefficient Magnitude\"\n        \n        if importance_scores is not None:\n            # PCA component importance\n            n_components = len(importance_scores)\n            component_names = [f'PC{i+1}' for i in range(n_components)]\n            \n            # Sort by importance\n            sorted_idx = np.argsort(importance_scores)[-20:]  # Top 20\n            \n            plt.figure(figsize=(12, 8))\n            \n            plt.subplot(2, 2, 1)\n            plt.barh(range(len(sorted_idx)), importance_scores[sorted_idx])\n            plt.yticks(range(len(sorted_idx)), [component_names[i] for i in sorted_idx])\n            plt.xlabel(f'{importance_type}')\n            plt.title(f'Top 20 Most Important PCA Components\\n{model_name}')\n            plt.grid(True, alpha=0.3)\n            \n            # Cumulative importance\n            plt.subplot(2, 2, 2)\n            sorted_importance = np.sort(importance_scores)[::-1]\n            cumulative_importance = np.cumsum(sorted_importance) / np.sum(sorted_importance)\n            \n            plt.plot(range(1, len(cumulative_importance) + 1), cumulative_importance)\n            plt.xlabel('Number of Components')\n            plt.ylabel('Cumulative Importance')\n            plt.title('Cumulative Feature Importance')\n            plt.grid(True, alpha=0.3)\n            plt.axhline(y=0.8, color='r', linestyle='--', label='80% Threshold')\n            plt.axhline(y=0.9, color='orange', linestyle='--', label='90% Threshold')\n            plt.legend()\n            \n            # Top 10 detailed\n            plt.subplot(2, 2, 3)\n            top_10_idx = sorted_idx[-10:]\n            plt.pie(importance_scores[top_10_idx], \n                   labels=[component_names[i] for i in top_10_idx],\n                   autopct='%1.1f%%', startangle=90)\n            plt.title('Distribution of Top 10 Most Important Components')\n            \n            # Statistics\n            plt.subplot(2, 2, 4)\n            plt.axis('off')\n            \n            # How many components for 80% and 90% importance\n            comp_80 = np.argmax(cumulative_importance >= 0.8) + 1\n            comp_90 = np.argmax(cumulative_importance >= 0.9) + 1\n            \n            stats_text = f\"\"\"FEATURE IMPORTANCE STATISTICS\n\nModel: {model_name}\nImportance Metric: {importance_type}\n\nComponent Statistics:\n• Total components: {len(importance_scores)}\n• For 80% importance: {comp_80} components\n• For 90% importance: {comp_90} components\n\nMost important component: {component_names[sorted_idx[-1]]}\nImportance: {importance_scores[sorted_idx[-1]]:.4f}\n\nImportance distribution:\n• Maximum: {importance_scores.max():.4f}\n• Average: {importance_scores.mean():.4f}\n• Minimum: {importance_scores.min():.4f}\"\"\"\n            \n            plt.text(0.1, 0.5, stats_text, transform=plt.gca().transAxes,\n                     fontsize=10, verticalalignment='center', fontfamily='monospace')\n            \n            plt.tight_layout()\n            plt.savefig(f'feature_importance_{model_name.lower()}.png', dpi=300, bbox_inches='tight')\n            plt.show()\n            \n            print(f\"✓ Feature importance analysis completed for {model_name}\")\n            print(f\"   Most important component: {component_names[sorted_idx[-1]]} ({importance_scores[sorted_idx[-1]]:.4f})\")\n            print(f\"   {comp_80} components sufficient for 80% importance\")\n            \n        else:\n            print(f\"⚠️ Feature importance analysis not supported for {model_name}\")\n            \n    except Exception as e:\n        print(f\"❌ Feature importance analysis error: {e}\")\n\n##### OVERFITTING ANALYSIS #####\n\"\"\"\nAşırı öğrenme analizi\n\"\"\"\ndef analyze_overfitting(model, X_train, y_train, X_test, y_test, model_name):\n    \"\"\"Analyze potential overfitting/underfitting\"\"\"\n    print_section(f\"OVERFITTING ANALYSIS - {model_name}\")\n    \n    try:\n        # Learning curves\n        train_sizes, train_scores, val_scores = learning_curve(\n            model, X_train, y_train, cv=3,\n            train_sizes=np.linspace(0.1, 1.0, 10),\n            scoring='accuracy', n_jobs=-1, random_state=CFG.seed\n        )\n        \n        train_mean = np.mean(train_scores, axis=1)\n        train_std = np.std(train_scores, axis=1)\n        val_mean = np.mean(val_scores, axis=1)\n        val_std = np.std(val_scores, axis=1)\n        \n        # Get final scores\n        train_final_accuracy = accuracy_score(y_train, model.predict(X_train))\n        test_final_accuracy = accuracy_score(y_test, model.predict(X_test))\n        \n        # Determine overfitting status\n        overfitting_gap = train_final_accuracy - test_final_accuracy\n        \n        if overfitting_gap > 0.1:\n            status = \"🔴 OVERFITTING\"\n            status_color = 'red'\n        elif overfitting_gap > 0.05:\n            status = \"🟡 MILD OVERFITTING\"\n            status_color = 'orange'\n        elif test_final_accuracy < 0.3:\n            status = \"🔵 UNDERFITTING\"\n            status_color = 'blue'\n        else:\n            status = \"🟢 BALANCED LEARNING\"\n            status_color = 'green'\n        \n        plt.figure(figsize=(15, 10))\n        \n        # Learning curve\n        plt.subplot(2, 3, 1)\n        plt.plot(train_sizes, train_mean, 'o-', color='blue', label='Training Score')\n        plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, alpha=0.1, color='blue')\n        plt.plot(train_sizes, val_mean, 'o-', color='red', label='Validation Score')\n        plt.fill_between(train_sizes, val_mean - val_std, val_mean + val_std, alpha=0.1, color='red')\n        plt.xlabel('Training Set Size')\n        plt.ylabel('Accuracy Score')\n        plt.title('Learning Curve')\n        plt.legend()\n        plt.grid(True, alpha=0.3)\n        \n        # Performance gap visualization\n        plt.subplot(2, 3, 2)\n        gap_values = train_mean - val_mean\n        plt.plot(train_sizes, gap_values, 'o-', color='purple')\n        plt.axhline(y=0.1, color='red', linestyle='--', label='Overfitting threshold')\n        plt.axhline(y=0.05, color='orange', linestyle='--', label='Acceptable threshold')\n        plt.xlabel('Training Set Size')\n        plt.ylabel('Training - Validation Gap')\n        plt.title('Performance Gap')\n        plt.legend()\n        plt.grid(True, alpha=0.3)\n        \n        # Final comparison\n        plt.subplot(2, 3, 3)\n        labels = ['Training', 'Test']\n        scores = [train_final_accuracy, test_final_accuracy]\n        colors = ['blue', 'red']\n        \n        bars = plt.bar(labels, scores, color=colors, alpha=0.7)\n        plt.ylabel('Accuracy')\n        plt.title('Final Performance Comparison')\n        plt.ylim(0, 1)\n        \n        # Add value labels on bars\n        for bar, score in zip(bars, scores):\n            height = bar.get_height()\n            plt.text(bar.get_x() + bar.get_width()/2., height + 0.01,\n                     f'{score:.3f}', ha='center', va='bottom')\n        \n        plt.grid(True, alpha=0.3)\n        \n        # Validation curve for key hyperparameter (if applicable)\n        plt.subplot(2, 3, 4)\n        try:\n            if hasattr(model.named_steps['classifier'], 'C'):  # Logistic Regression\n                param_name = 'classifier__C'\n                param_range = [0.01, 0.1, 1, 10, 100]\n            elif hasattr(model.named_steps['classifier'], 'n_estimators'):  # Random Forest\n                param_name = 'classifier__n_estimators'\n                param_range = [10, 50, 100, 200, 500]\n            elif hasattr(model.named_steps['classifier'], 'n_neighbors'):  # KNN\n                param_name = 'classifier__n_neighbors'\n                param_range = [1, 3, 5, 7, 9, 11]\n            else:\n                param_name = None\n                \n            if param_name:\n                train_scores_val, test_scores_val = validation_curve(\n                    model, X_train, y_train, param_name=param_name,\n                    param_range=param_range, cv=3, scoring='accuracy', n_jobs=-1\n                )\n                \n                train_mean_val = np.mean(train_scores_val, axis=1)\n                test_mean_val = np.mean(test_scores_val, axis=1)\n                \n                plt.plot(param_range, train_mean_val, 'o-', color='blue', label='Training')\n                plt.plot(param_range, test_mean_val, 'o-', color='red', label='Validation')\n                plt.xlabel(param_name.split('__')[1])\n                plt.ylabel('Accuracy')\n                plt.title('Validation Curve')\n                plt.legend()\n                plt.grid(True, alpha=0.3)\n                if param_name == 'classifier__C':\n                    plt.xscale('log')\n            else:\n                plt.text(0.5, 0.5, 'Validation curve\\nnot available\\nfor this model', \n                         ha='center', va='center', transform=plt.gca().transAxes)\n                plt.axis('off')\n                \n        except Exception as e:\n            plt.text(0.5, 0.5, f'Validation curve\\nerror:\\n{str(e)[:50]}...', \n                     ha='center', va='center', transform=plt.gca().transAxes)\n            plt.axis('off')\n        \n        # Recommendations\n        plt.subplot(2, 3, 5)\n        plt.axis('off')\n        \n        # Generate recommendations\n        recommendations = []\n        if overfitting_gap > 0.1:\n            recommendations = [\n                \"• Collect more training data\",\n                \"• Increase regularization parameters\",\n                \"• Reduce model complexity\",\n                \"• Use dropout or early stopping\",\n                \"• Tune parameters with cross-validation\"\n            ]\n        elif overfitting_gap > 0.05:\n            recommendations = [\n                \"• Slightly increase regularization\",\n                \"• Use more cross-validation\",\n                \"• Apply feature selection\"\n            ]\n        elif test_final_accuracy < 0.3:\n            recommendations = [\n                \"• Increase model complexity\",\n                \"• Add more features\",\n                \"• Try different model architecture\",\n                \"• Improve data preprocessing\"\n            ]\n        else:\n            recommendations = [\n                \"• Model performance is balanced\",\n                \"• Current configuration is suitable\",\n                \"• Optional fine-tuning possible\"\n            ]\n        \n        rec_text = f\"\"\"MODEL STATUS\n{status}\n\nPerformance Metrics:\n• Training Accuracy: {train_final_accuracy:.4f}\n• Test Accuracy: {test_final_accuracy:.4f}\n• Performance Gap: {overfitting_gap:.4f}\n\nRECOMMENDATIONS:\n\"\"\" + \"\\n\".join(recommendations)\n        \n        plt.text(0.05, 0.95, rec_text, transform=plt.gca().transAxes,\n                 fontsize=10, verticalalignment='top', fontfamily='monospace')\n        \n        # Summary metrics\n        plt.subplot(2, 3, 6)\n        plt.axis('off')\n        \n        # Calculate additional metrics\n        final_train_val_gap = train_mean[-1] - val_mean[-1]\n        learning_efficiency = (val_mean[-1] - val_mean[0]) / (train_sizes[-1] - train_sizes[0])\n        \n        metrics_text = f\"\"\"DETAILED METRICS\n\nLearning Curve:\n• Initial CV score: {val_mean[0]:.4f}\n• Final CV score: {val_mean[-1]:.4f}\n• Learning efficiency: {learning_efficiency:.6f}\n\nOverfitting Signals:\n• Train-Test gap: {overfitting_gap:.4f}\n• Train-CV gap: {final_train_val_gap:.4f}\n\nModel Stability:\n• CV standard deviation: {val_std[-1]:.4f}\n• Training standard deviation: {train_std[-1]:.4f}\"\"\"\n        \n        plt.text(0.05, 0.95, metrics_text, transform=plt.gca().transAxes,\n                 fontsize=10, verticalalignment='top', fontfamily='monospace')\n        \n        plt.tight_layout()\n        plt.savefig(f'overfitting_analysis_{model_name.lower()}.png', dpi=300, bbox_inches='tight')\n        plt.show()\n        \n        # Log results\n        print(f\"📊 {model_name} Overfitting Analysis:\")\n        print(f\"   {status}\")\n        print(f\"   Training Accuracy: {train_final_accuracy:.4f}\")\n        print(f\"   Test Accuracy: {test_final_accuracy:.4f}\")\n        print(f\"   Performance Gap: {overfitting_gap:.4f}\")\n        \n        return {\n            'status': status,\n            'train_accuracy': train_final_accuracy,\n            'test_accuracy': test_final_accuracy,\n            'overfitting_gap': overfitting_gap,\n            'recommendations': recommendations\n        }\n        \n    except Exception as e:\n        print(f\"❌ Overfitting analysis error: {e}\")\n        return None\n\n##### NEURAL NETWORK TRAINING #####\n\"\"\"\nNeural Network training with different architectures\n\"\"\"\ndef create_neural_network(input_dim, num_classes, architecture, dropout_rate=0.2):\n    \"\"\"Create neural network with specified architecture and GPU optimizations\"\"\"\n    if not TENSORFLOW_AVAILABLE:\n        return None\n    \n    model = keras.Sequential()\n    \n    # Input layer with optimized initialization\n    model.add(layers.Dense(\n        architecture[0], \n        activation='relu', \n        input_shape=(input_dim,),\n        kernel_initializer='he_normal',  # Better for ReLU\n        bias_initializer='zeros'\n    ))\n    model.add(layers.BatchNormalization())\n    \n    # Hidden layers with optimizations\n    for i in range(1, len(architecture)):\n        model.add(layers.Dense(\n            architecture[i], \n            activation='relu',\n            kernel_initializer='he_normal',\n            bias_initializer='zeros'\n        ))\n        model.add(layers.BatchNormalization())\n        if dropout_rate > 0:\n            model.add(layers.Dropout(dropout_rate))\n    \n    # Output layer\n    if num_classes > 2:\n        model.add(layers.Dense(num_classes, activation='softmax', name='predictions'))\n        loss = 'categorical_crossentropy'\n    else:\n        model.add(layers.Dense(1, activation='sigmoid', name='predictions'))\n        loss = 'binary_crossentropy'\n    \n    # Mixed precision optimization for output layer\n    if tf.config.list_physical_devices('GPU') and hasattr(tf.keras.mixed_precision, 'Policy'):\n        # Cast to float32 for numerical stability in mixed precision\n        model.add(layers.Activation('linear', dtype='float32'))\n    \n    # Compile model with optimized settings\n    optimizer = keras.optimizers.Adam(\n        learning_rate=0.001,\n        beta_1=0.9,\n        beta_2=0.999,\n        epsilon=1e-7,  # Better for mixed precision\n        amsgrad=False\n    )\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=loss,\n        metrics=['accuracy'],\n        # Enable XLA compilation if available\n        jit_compile=True if tf.config.list_physical_devices('GPU') else False\n    )\n    \n    return model, loss\n\ndef train_neural_networks(X_train, y_train, X_test, y_test, num_classes, cfg):\n    \"\"\"Train different neural network architectures with GPU optimizations and overfitting detection\"\"\"\n    if not TENSORFLOW_AVAILABLE:\n        print(\"⚠️ TensorFlow not available. Skipping neural network training.\")\n        return []\n    \n    print_section(\"NEURAL NETWORK TRAINING\")\n    \n    # Define different architectures to test - with optimization for debug mode\n    if hasattr(cfg, 'reduce_model_architectures') and cfg.reduce_model_architectures:\n        print(\"🔬 Debug mode: Using reduced architecture set for faster testing\")\n        architectures = [\n            [128, 64],\n            [256, 128, 64],\n            [128, 128, 64]\n        ]\n    else:\n        architectures = [\n            [128, 64],\n            [256, 128, 64],\n            [128, 128, 64],\n            [512, 256, 128, 64],\n            [64, 32],\n            [256, 128],\n            [128, 64, 32],\n            [512, 256, 128],\n            [1024, 512, 256, 128],\n            [256, 256, 128, 64]\n        ]\n    \n    # GPU Optimization settings\n    batch_size = getattr(cfg, 'batch_size_neural_networks', 64)\n    epochs = getattr(cfg, 'neural_network_epochs', 50)\n    patience = getattr(cfg, 'early_stopping_patience', 5)\n    \n    # Overfitting detection settings\n    overfitting_threshold = getattr(cfg, 'overfitting_threshold', 0.15)\n    max_overfitting_models = getattr(cfg, 'max_overfitting_models', 3)\n    \n    print(f\"⚡ GPU Optimization Settings:\")\n    print(f\"   • Batch size: {batch_size} (optimized for GPU)\")\n    print(f\"   • Max epochs: {epochs}\")\n    print(f\"   • Early stopping patience: {patience}\")\n    print(f\"   • Mixed precision: {getattr(cfg, 'enable_mixed_precision', True)}\")\n    print(f\"   • XLA compilation: {getattr(cfg, 'enable_xla', True)}\")\n    \n    print(f\"🔍 Overfitting Detection Settings:\")\n    print(f\"   • Overfitting threshold: {overfitting_threshold} (max acceptable train-val loss gap)\")\n    print(f\"   • Max overfitting models to skip: {max_overfitting_models}\")\n    print(f\"   • Strategy: Skip overfitting models and continue until balanced model found\")\n    \n    # Prepare data for neural networks\n    if num_classes > 2:\n        y_train_cat = to_categorical(y_train, num_classes)\n        y_test_cat = to_categorical(y_test, num_classes)\n    else:\n        y_train_cat = y_train\n        y_test_cat = y_test\n    \n    # Scale features for neural networks - using float32 for GPU efficiency\n    print(\"🔄 Scaling features with float32 precision for GPU efficiency...\")\n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train).astype(np.float32)\n    X_test_scaled = scaler.transform(X_test).astype(np.float32)\n    \n    # Create TensorFlow datasets for better GPU utilization\n    print(\"📊 Creating optimized TensorFlow datasets...\")\n    \n    def create_tf_dataset(X, y, batch_size, is_training=True):\n        dataset = tf.data.Dataset.from_tensor_slices((X, y))\n        if is_training:\n            dataset = dataset.shuffle(buffer_size=1000)\n        dataset = dataset.batch(batch_size)\n        dataset = dataset.prefetch(tf.data.AUTOTUNE)\n        return dataset\n    \n    train_dataset = create_tf_dataset(X_train_scaled, y_train_cat, batch_size, is_training=True)\n    test_dataset = create_tf_dataset(X_test_scaled, y_test_cat, batch_size, is_training=False)\n    \n    results = []\n    overfitting_count = 0\n    successful_models = 0\n    \n    print(f\"🧠 Training {len(architectures)} different neural network architectures...\")\n    print(f\"📊 Input dimension: {X_train.shape[1]}, Output classes: {num_classes}\")\n    print(\"=\" * 80)\n    \n    for i, arch in enumerate(architectures):\n        print(f\"\\n🔄 Training Architecture {i+1}/{len(architectures)}: {'/'.join(map(str, arch))}\")\n        \n        try:\n            # Create model with optimizations\n            model, loss_type = create_neural_network(X_train.shape[1], num_classes, arch)\n            \n            if model is None:\n                continue\n            \n            # Enhanced callbacks for GPU optimization\n            callbacks = []\n            \n            # Early stopping\n            early_stopping = keras.callbacks.EarlyStopping(\n                monitor='val_loss',\n                patience=patience,\n                restore_best_weights=True,\n                verbose=0\n            )\n            callbacks.append(early_stopping)\n            \n            # Learning rate reduction on plateau\n            lr_reducer = keras.callbacks.ReduceLROnPlateau(\n                monitor='val_loss',\n                factor=0.5,\n                patience=max(2, patience//2),\n                min_lr=1e-7,\n                verbose=0\n            )\n            callbacks.append(lr_reducer)\n            \n            # Mixed precision loss scaling (if enabled)\n            if getattr(cfg, 'enable_mixed_precision', True) and tf.config.list_physical_devices('GPU'):\n                # Model already uses mixed precision policy from global setting\n                pass\n            \n            # Train model with optimized settings\n            start_time = time.time()\n            \n            with tf.device('/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'):\n                history = model.fit(\n                    train_dataset,\n                    validation_data=test_dataset,\n                    epochs=epochs,\n                    callbacks=callbacks,\n                    verbose=0,\n                    # Additional GPU optimizations\n                    steps_per_epoch=len(X_train) // batch_size,\n                    validation_steps=len(X_test) // batch_size\n                )\n            \n            training_time = time.time() - start_time\n            \n            # Get final training and validation losses for overfitting detection\n            final_train_loss = history.history['loss'][-1]\n            final_val_loss = history.history['val_loss'][-1]\n            loss_gap = final_val_loss - final_train_loss\n            \n            # Overfitting detection with detailed explanation\n            is_overfitting = loss_gap > overfitting_threshold\n            \n            if is_overfitting:\n                overfitting_count += 1\n                print(f\"   🚨 OVERFITTING DETECTED!\")\n                print(f\"   📊 Train Loss: {final_train_loss:.4f}\")\n                print(f\"   📊 Validation Loss: {final_val_loss:.4f}\")\n                print(f\"   📊 Loss Gap: {loss_gap:.4f} (threshold: {overfitting_threshold})\")\n                print(f\"   💡 Since there is a massive gap between validation loss ({final_val_loss:.4f}) and train loss ({final_train_loss:.4f}), this indicates that the model is overfitting. Skipping this model.\")\n                \n                # Check if we should stop trying more models\n                if overfitting_count >= max_overfitting_models:\n                    print(f\"   ⛔ Reached maximum overfitting models limit ({max_overfitting_models}). Continuing to search for balanced model...\")\n                \n                # Clean up and skip this model\n                del model\n                tf.keras.backend.clear_session()\n                import gc\n                gc.collect()\n                continue\n            \n            # Model passed overfitting check - evaluate it\n            print(f\"   ✅ Model passed overfitting check (gap: {loss_gap:.4f} < {overfitting_threshold})\")\n            \n            # Evaluate model\n            test_loss, test_accuracy = model.evaluate(test_dataset, verbose=0)\n            train_loss, train_accuracy = model.evaluate(train_dataset, verbose=0)\n            \n            # Get predictions for additional metrics\n            print(f\"   📊 Computing detailed metrics...\")\n            y_pred_probs = model.predict(test_dataset, verbose=0)\n            \n            if num_classes > 2:\n                y_pred = np.argmax(y_pred_probs, axis=1)\n                y_true = np.argmax(y_test_cat, axis=1)\n            else:\n                y_pred = (y_pred_probs > 0.5).astype(int).flatten()\n                y_true = y_test_cat\n            \n            # Calculate additional metrics\n            precision, recall, f1, _ = precision_recall_fscore_support(y_true, y_pred, average='weighted', zero_division=0)\n            \n            # Store results\n            arch_str = '/'.join(map(str, arch + [num_classes]))\n            results.append({\n                'Architecture': arch_str,\n                'Parameters': model.count_params(),\n                'Train_Accuracy': train_accuracy,\n                'Test_Accuracy': test_accuracy,\n                'Train_Loss': train_loss,\n                'Test_Loss': test_loss,\n                'Loss_Gap': loss_gap,\n                'Precision': precision,\n                'Recall': recall,\n                'F1_Score': f1,\n                'Training_Time': training_time,\n                'Epochs_Trained': len(history.history['loss']),\n                'Overfitting_Gap': train_accuracy - test_accuracy,\n                'Model': model,\n                'History': history\n            })\n            \n            successful_models += 1\n            print(f\"   ✓ Acc: {test_accuracy:.4f}, Loss: {test_loss:.4f}, Time: {training_time:.1f}s, Epochs: {len(history.history['loss'])}\")\n            \n            # Memory cleanup between models\n            if getattr(cfg, 'clear_memory_between_models', True):\n                del model\n                tf.keras.backend.clear_session()\n                # Force garbage collection\n                import gc\n                gc.collect()\n            \n        except Exception as e:\n            print(f\"   ❌ Error training architecture {arch}: {e}\")\n            # Cleanup on error\n            try:\n                tf.keras.backend.clear_session()\n                import gc\n                gc.collect()\n            except:\n                pass\n            continue\n    \n    # Summary of training process\n    print(f\"\\n📊 Training Summary:\")\n    print(f\"   • Total architectures attempted: {len(architectures)}\")\n    print(f\"   • Successful models: {successful_models}\")\n    print(f\"   • Overfitting models skipped: {overfitting_count}\")\n    print(f\"   • Success rate: {(successful_models/len(architectures)*100):.1f}%\")\n    \n    if not results:\n        print(\"❌ No neural networks were successfully trained!\")\n        if overfitting_count > 0:\n            print(f\"💡 All {overfitting_count} models showed overfitting. Consider:\")\n            print(\"   • Reducing model complexity\")\n            print(\"   • Adding more regularization\")\n            print(\"   • Collecting more training data\")\n            print(\"   • Increasing dropout rate\")\n        return []\n    \n    # Display results table\n    print_section(\"NEURAL NETWORK RESULTS\")\n    \n    # Performance statistics\n    total_training_time = sum(r['Training_Time'] for r in results)\n    avg_time_per_model = total_training_time / len(results)\n    \n    print(f\"⚡ Performance Summary:\")\n    print(f\"   • Total training time: {total_training_time:.1f}s ({total_training_time/60:.1f}m)\")\n    print(f\"   • Average time per model: {avg_time_per_model:.1f}s\")\n    print(f\"   • Models trained: {len(results)}\")\n    \n    # Create results DataFrame with loss gap information\n    results_df = pd.DataFrame([{\n        'Architecture': r['Architecture'],\n        'Parameters': f\"{r['Parameters']:,}\",\n        'Test_Accuracy': f\"{r['Test_Accuracy']:.4f}\",\n        'Test_Loss': f\"{r['Test_Loss']:.4f}\",\n        'Loss_Gap': f\"{r['Loss_Gap']:.4f}\",\n        'Precision': f\"{r['Precision']:.4f}\",\n        'F1_Score': f\"{r['F1_Score']:.4f}\",\n        'Overfitting': f\"{r['Overfitting_Gap']:.4f}\",\n        'Time(s)': f\"{r['Training_Time']:.1f}\",\n        'Epochs': r['Epochs_Trained']\n    } for r in results])\n    \n    # Sort by test accuracy\n    results_df = results_df.sort_values('Test_Accuracy', ascending=False)\n    \n    print(\"🏆 NEURAL NETWORK PERFORMANCE TABLE\")\n    print(\"=\" * 130)\n    print(results_df.to_string(index=False))\n    print(\"=\" * 130)\n    \n    # Find best model\n    best_result = max(results, key=lambda x: x['Test_Accuracy'])\n    print(f\"\\n🥇 BEST NEURAL NETWORK:\")\n    print(f\"   Architecture: {best_result['Architecture']}\")\n    print(f\"   Test Accuracy: {best_result['Test_Accuracy']:.4f}\")\n    print(f\"   Test Loss: {best_result['Test_Loss']:.4f}\")\n    print(f\"   Loss Gap: {best_result['Loss_Gap']:.4f}\")\n    print(f\"   Parameters: {best_result['Parameters']:,}\")\n    print(f\"   Training Time: {best_result['Training_Time']:.1f} seconds\")\n    print(f\"   Epochs Trained: {best_result['Epochs_Trained']}\")\n    \n    # Visualize results\n    visualize_neural_network_results(results)\n    \n    return results\n\ndef visualize_neural_network_results(results):\n    \"\"\"Visualize neural network training results\"\"\"\n    if not results:\n        return\n    \n    plt.figure(figsize=(20, 15))\n    \n    # 1. Accuracy comparison\n    plt.subplot(3, 4, 1)\n    architectures = [r['Architecture'] for r in results]\n    test_accuracies = [r['Test_Accuracy'] for r in results]\n    train_accuracies = [r['Train_Accuracy'] for r in results]\n    \n    x = range(len(architectures))\n    width = 0.35\n    \n    plt.bar([i - width/2 for i in x], train_accuracies, width, label='Train', alpha=0.8)\n    plt.bar([i + width/2 for i in x], test_accuracies, width, label='Test', alpha=0.8)\n    plt.xlabel('Architecture')\n    plt.ylabel('Accuracy')\n    plt.title('Train vs Test Accuracy')\n    plt.xticks(x, [arch.replace('/', '\\n') for arch in architectures], rotation=45, fontsize=8)\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    \n    # 2. Loss comparison\n    plt.subplot(3, 4, 2)\n    test_losses = [r['Test_Loss'] for r in results]\n    train_losses = [r['Train_Loss'] for r in results]\n    \n    plt.bar([i - width/2 for i in x], train_losses, width, label='Train', alpha=0.8)\n    plt.bar([i + width/2 for i in x], test_losses, width, label='Test', alpha=0.8)\n    plt.xlabel('Architecture')\n    plt.ylabel('Loss')\n    plt.title('Train vs Test Loss')\n    plt.xticks(x, [arch.replace('/', '\\n') for arch in architectures], rotation=45, fontsize=8)\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    \n    # 3. Parameter count vs accuracy\n    plt.subplot(3, 4, 3)\n    param_counts = [r['Parameters'] for r in results]\n    plt.scatter(param_counts, test_accuracies, alpha=0.7, s=100)\n    plt.xlabel('Number of Parameters')\n    plt.ylabel('Test Accuracy')\n    plt.title('Parameters vs Accuracy')\n    plt.grid(True, alpha=0.3)\n    \n    # Add labels for each point\n    for i, arch in enumerate(architectures):\n        plt.annotate(arch.replace('/', '\\n'), (param_counts[i], test_accuracies[i]), \n                    xytext=(5, 5), textcoords='offset points', fontsize=6)\n    \n    # 4. Training time vs accuracy\n    plt.subplot(3, 4, 4)\n    training_times = [r['Training_Time'] for r in results]\n    plt.scatter(training_times, test_accuracies, alpha=0.7, s=100)\n    plt.xlabel('Training Time (seconds)')\n    plt.ylabel('Test Accuracy')\n    plt.title('Training Time vs Accuracy')\n    plt.grid(True, alpha=0.3)\n    \n    # 5. Overfitting analysis\n    plt.subplot(3, 4, 5)\n    overfitting_gaps = [r['Overfitting_Gap'] for r in results]\n    colors = ['red' if gap > 0.1 else 'orange' if gap > 0.05 else 'green' for gap in overfitting_gaps]\n    \n    plt.bar(x, overfitting_gaps, color=colors, alpha=0.7)\n    plt.xlabel('Architecture')\n    plt.ylabel('Overfitting Gap')\n    plt.title('Overfitting Analysis')\n    plt.xticks(x, [arch.replace('/', '\\n') for arch in architectures], rotation=45, fontsize=8)\n    plt.axhline(y=0.1, color='red', linestyle='--', label='High overfitting')\n    plt.axhline(y=0.05, color='orange', linestyle='--', label='Mild overfitting')\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    \n    # 6. F1 Score comparison\n    plt.subplot(3, 4, 6)\n    f1_scores = [r['F1_Score'] for r in results]\n    plt.bar(x, f1_scores, alpha=0.8, color='purple')\n    plt.xlabel('Architecture')\n    plt.ylabel('F1 Score')\n    plt.title('F1 Score Comparison')\n    plt.xticks(x, [arch.replace('/', '\\n') for arch in architectures], rotation=45, fontsize=8)\n    plt.grid(True, alpha=0.3)\n    \n    # 7. Training curves for best model\n    plt.subplot(3, 4, 7)\n    best_result = max(results, key=lambda x: x['Test_Accuracy'])\n    history = best_result['History']\n    \n    plt.plot(history.history['accuracy'], label='Train Accuracy')\n    plt.plot(history.history['val_accuracy'], label='Val Accuracy')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.title(f'Best Model Training Curve\\n{best_result[\"Architecture\"]}')\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    \n    # 8. Loss curves for best model\n    plt.subplot(3, 4, 8)\n    plt.plot(history.history['loss'], label='Train Loss')\n    plt.plot(history.history['val_loss'], label='Val Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title(f'Best Model Loss Curve\\n{best_result[\"Architecture\"]}')\n    plt.legend()\n    plt.grid(True, alpha=0.3)\n    \n    # 9. Epochs trained\n    plt.subplot(3, 4, 9)\n    epochs_trained = [r['Epochs_Trained'] for r in results]\n    plt.bar(x, epochs_trained, alpha=0.8, color='teal')\n    plt.xlabel('Architecture')\n    plt.ylabel('Epochs Trained')\n    plt.title('Training Epochs (Early Stopping)')\n    plt.xticks(x, [arch.replace('/', '\\n') for arch in architectures], rotation=45, fontsize=8)\n    plt.grid(True, alpha=0.3)\n    \n    # 10. Precision vs Recall\n    plt.subplot(3, 4, 10)\n    precisions = [r['Precision'] for r in results]\n    recalls = [r['Recall'] for r in results]\n    plt.scatter(recalls, precisions, alpha=0.7, s=100)\n    plt.xlabel('Recall')\n    plt.ylabel('Precision')\n    plt.title('Precision vs Recall')\n    plt.grid(True, alpha=0.3)\n    \n    # Add labels\n    for i, arch in enumerate(architectures):\n        plt.annotate(arch, (recalls[i], precisions[i]), \n                    xytext=(5, 5), textcoords='offset points', fontsize=6)\n    \n    # 11. Performance summary table\n    plt.subplot(3, 4, 11)\n    plt.axis('off')\n    \n    # Sort results by accuracy\n    sorted_results = sorted(results, key=lambda x: x['Test_Accuracy'], reverse=True)\n    \n    summary_text = \"TOP 5 MODELS:\\n\\n\"\n    for i, r in enumerate(sorted_results[:5]):\n        summary_text += f\"{i+1}. {r['Architecture']}\\n\"\n        summary_text += f\"   Acc: {r['Test_Accuracy']:.4f}\\n\"\n        summary_text += f\"   Loss: {r['Test_Loss']:.4f}\\n\"\n        summary_text += f\"   Params: {r['Parameters']:,}\\n\\n\"\n    \n    plt.text(0.1, 0.9, summary_text, transform=plt.gca().transAxes,\n             fontsize=10, verticalalignment='top', fontfamily='monospace')\n    \n    # 12. Performance statistics\n    plt.subplot(3, 4, 12)\n    plt.axis('off')\n    \n    stats_text = f\"\"\"NEURAL NETWORK STATISTICS\n\nTotal models trained: {len(results)}\n\nAccuracy Statistics:\n• Best: {max(test_accuracies):.4f}\n• Worst: {min(test_accuracies):.4f}\n• Average: {np.mean(test_accuracies):.4f}\n• Std: {np.std(test_accuracies):.4f}\n\nParameter Statistics:\n• Largest: {max(param_counts):,}\n• Smallest: {min(param_counts):,}\n• Average: {int(np.mean(param_counts)):,}\n\nTraining Time:\n• Total: {sum(training_times):.1f}s\n• Average: {np.mean(training_times):.1f}s\"\"\"\n    \n    plt.text(0.1, 0.9, stats_text, transform=plt.gca().transAxes,\n             fontsize=10, verticalalignment='top', fontfamily='monospace')\n    \n    plt.tight_layout()\n    plt.savefig('neural_network_results.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    print(\"✓ Neural network results visualization saved as 'neural_network_results.png'\")\n\n##### MAIN PIPELINE #####\n\"\"\"\nAna çalışma pipeline'ı\n\"\"\"\ndef main():\n    \"\"\"Enhanced main pipeline with all new features\"\"\"\n    setup_logging()\n    set_seed(CFG.seed)\n    \n    print_section(\"🚀 BirdCLEF ML Pipeline Starting\")\n    \n    # Display configuration\n    print_configuration(CFG)\n    \n    total_start_time = time.time()\n    \n    try:\n        # 1. Data Preparation\n        train_df, label_encoder = prepare_data(CFG)\n        \n        # 2. Feature Extraction\n        X, y = extract_features(train_df, CFG)\n        \n        # 3. Apply Mixup Augmentation\n        if CFG.enable_data_augmentation:\n            X, y = apply_mixup_features(X, y, CFG)\n        \n        # 4. Train-Test Split\n        print_section(\"DATA SPLITTING\")\n        print(\"📊 Splitting into training and test sets...\")\n        \n        # Check if stratification is possible\n        unique, counts = np.unique(y, return_counts=True)\n        min_class_count = counts.min()\n        \n        if min_class_count >= 2:\n            X_train, X_test, y_train, y_test = train_test_split(\n                X, y, test_size=CFG.test_size, random_state=CFG.seed, \n                stratify=y\n            )\n            print(\"✓ Stratified split used\")\n        else:\n            X_train, X_test, y_train, y_test = train_test_split(\n                X, y, test_size=CFG.test_size, random_state=CFG.seed\n            )\n            print(\"⚠️ Stratified split not possible (insufficient samples)\")\n        \n        print(f\"✓ Training: {X_train.shape}, Test: {X_test.shape}\")\n        \n        # 5. PCA Analysis\n        optimal_n_components = analyze_pca_components(X_train, CFG)\n        X_train_reduced, X_test_reduced, pca = apply_pca_transformation(\n            X_train, X_test, optimal_n_components, CFG\n        )\n        \n        # 6. Model Training\n        best_model, best_model_name, all_results = train_and_evaluate_models(\n            X_train_reduced, y_train, X_test_reduced, y_test, CFG, label_encoder\n        )\n        \n        if best_model is None:\n            print(\"❌ No model was successfully trained!\")\n            return\n        \n        # 7. Detailed Evaluation\n        accuracy, precision, recall, f1 = detailed_model_evaluation(\n            best_model, X_test_reduced, y_test, label_encoder, best_model_name\n        )\n        \n        # 8. Pseudo-labeling (if enabled)\n        if CFG.enable_pseudo_labeling:\n            soundscape_files = extract_soundscape_files(CFG)\n            if soundscape_files:\n                pseudo_X, pseudo_y = generate_pseudo_labels(best_model, soundscape_files, CFG, label_encoder)\n                \n                if len(pseudo_X) > 0:\n                    # Apply PCA to pseudo-features\n                    pseudo_X_reduced = pca.transform(pseudo_X)\n                    \n                    # Combine with original training data\n                    X_train_enhanced = np.vstack([X_train_reduced, pseudo_X_reduced])\n                    y_train_enhanced = np.concatenate([y_train, pseudo_y])\n                    \n                    print(\"🔄 Retraining best model with pseudo-labels...\")\n                    \n                    # Retrain the best model with enhanced data\n                    best_model.fit(X_train_enhanced, y_train_enhanced)\n                    \n                    # Evaluate enhanced model\n                    enhanced_accuracy = accuracy_score(y_test, best_model.predict(X_test_reduced))\n                    print(f\"📊 Enhanced model accuracy: {enhanced_accuracy:.4f}\")\n                    accuracy = enhanced_accuracy  # Update final accuracy\n        \n        # 9. Feature Importance Analysis\n        analyze_feature_importance(best_model, best_model_name, pca)\n        \n        # 10. Overfitting Analysis\n        overfitting_results = analyze_overfitting(\n            best_model, X_train_reduced, y_train, X_test_reduced, y_test, best_model_name\n        )\n        \n        # 11. Neural Network Training\n        neural_network_results = train_neural_networks(X_train_reduced, y_train, X_test_reduced, y_test, CFG.num_classes, CFG)\n        \n        # 12. Save Best Model\n        print_section(\"MODEL SAVING\")\n        model_filename = f\"best_model_{best_model_name.lower()}_acc_{accuracy:.3f}.joblib\"\n        joblib.dump(best_model, model_filename)\n        print(f\"✓ Best model saved: {model_filename}\")\n        \n        # Save best neural network if available\n        if neural_network_results:\n            best_nn_result = max(neural_network_results, key=lambda x: x['Test_Accuracy'])\n            nn_filename = f\"best_neural_network_{best_nn_result['Architecture'].replace('/', '_')}_acc_{best_nn_result['Test_Accuracy']:.3f}.h5\"\n            best_nn_result['Model'].save(nn_filename)\n            print(f\"✓ Best neural network saved: {nn_filename}\")\n        \n        # 13. Final Summary\n        total_time = time.time() - total_start_time\n        \n        print_section(\"📋 SUMMARY REPORT\")\n        \n        # Neural network summary\n        nn_summary = \"\"\n        if neural_network_results:\n            best_nn = max(neural_network_results, key=lambda x: x['Test_Accuracy'])\n            nn_summary = f\"\"\"\n🧠 Neural Network Results:\n• Best Architecture: {best_nn['Architecture']}\n• Best NN Accuracy: {best_nn['Test_Accuracy']:.4f}\n• Best NN Loss: {best_nn['Test_Loss']:.4f}\n• Total NN Models Trained: {len(neural_network_results)}\n\"\"\"\n        else:\n            nn_summary = \"\\n🧠 Neural Networks: Not available (TensorFlow not installed)\"\n        \n        summary_text = f\"\"\"\n🎯 BirdCLEF Model Training Completed!\n\n⏱️  Total Time: {total_time:.2f} seconds ({total_time/60:.1f} minutes)\n\n📊 Data Information:\n• Total sample count: {len(train_df)}\n• Number of classes: {CFG.num_classes}\n• Feature dimensions: {X.shape[1]} → {optimal_n_components} (PCA)\n• Train/Test: {len(y_train)}/{len(y_test)}\n\n🏆 Best Traditional Model: {best_model_name}\n• Test Accuracy: {accuracy:.4f}\n• Precision: {precision:.4f}\n• Recall: {recall:.4f}\n• F1-Score: {f1:.4f}\n{nn_summary}\n📁 Saved Files:\n• Model: {model_filename}\n• Label Encoder: label_encoder.joblib\n• PCA Transformer: pca_transformer.joblib\n• Training Log: training_log.log\n\n📈 Visualizations:\n• PCA Analysis: pca_analysis.png\n• Model Evaluation: evaluation_{best_model_name.lower()}.png\n• Feature Importance: feature_importance_{best_model_name.lower()}.png\n• Overfitting Analysis: overfitting_analysis_{best_model_name.lower()}.png\n• Neural Network Results: neural_network_results.png\n\n⚙️ Configuration Used:\n• Data Augmentation: {'✓ Enabled' if CFG.enable_data_augmentation else '✗ Disabled'}\n• Pseudo-labeling: {'✓ Enabled' if CFG.enable_pseudo_labeling else '✗ Disabled'}\n• Quality Filtering: {'✓ Enabled' if CFG.filter_low_quality else '✗ Disabled'}\n        \"\"\"\n        \n        print(summary_text)\n        print(\"🎉 Pipeline completed successfully!\")\n        \n    except Exception as e:\n        print(f\"❌ Pipeline error: {e}\")\n        raise\n\nif __name__ == \"__main__\":\n    main() ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}