{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport soundfile as sf\nfrom pathlib import Path\nfrom glob import glob\nimport librosa\nprint(librosa.__version__)\nimport gc\nfrom concurrent.futures import ThreadPoolExecutor\nimport matplotlib.pyplot as plt\nimport librosa.display\nimport seaborn as sns\nimport plotly.express as px\nfrom sklearn.preprocessing import LabelEncoder, label_binarize\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import auc, accuracy_score, confusion_matrix, roc_auc_score, roc_curve, precision_recall_curve, classification_report, ConfusionMatrixDisplay \nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, Input, LSTM, Reshape\nfrom tensorflow.keras.utils import to_categorical\nfrom keras.layers import Layer\nfrom sklearn.impute import SimpleImputer\nfrom keras.callbacks import EarlyStopping\nimport keras_tuner as kt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-31T12:02:54.748232Z","iopub.execute_input":"2024-05-31T12:02:54.748694Z","iopub.status.idle":"2024-05-31T12:02:54.764361Z","shell.execute_reply.started":"2024-05-31T12:02:54.748658Z","shell.execute_reply":"2024-05-31T12:02:54.763119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation and Augmentation","metadata":{}},{"cell_type":"code","source":"metadata = pd.read_csv('/kaggle/input/birdclef-2024/train_metadata.csv')\nprint(metadata.head())\nprint(metadata.columns)\n\n# Function to find and sample .ogg files recursively in a given directory\ndef sample_ogg_files(directory, sample_size):\n    base_path = Path(directory)\n    all_ogg_files = list(base_path.rglob('*.ogg'))  # 递归地查找所有 .ogg 文件\n    if sample_size < len(all_ogg_files):\n        sampled_ogg_files = np.random.choice(all_ogg_files, sample_size, replace=False)\n    else:\n        sampled_ogg_files = all_ogg_files  # 如果样本大小大于文件总数，则返回所有文件\n    return sampled_ogg_files\n\n# Function to load and normalize audio, with optional augmentation\ndef load_and_normalize_audio(file_path, target_length=30, sr=22050, augment=False):\n    try:\n        audio, samplerate = sf.read(file_path, dtype='float32')\n        if len(audio.shape) == 2:\n            audio = np.mean(audio, axis=1)\n        audio = audio / np.max(np.abs(audio))\n        current_length = len(audio)\n        target_length_samples = sr * target_length\n        if current_length > target_length_samples:\n            audio = audio[:target_length_samples]\n        elif current_length < target_length_samples:\n            padding = np.zeros(target_length_samples - current_length)\n            audio = np.concatenate((audio, padding))\n\n        if augment:\n            audio = augment_audio(audio, sr)\n\n    except Exception as e:\n        print(f\"Error loading audio file {file_path}: {e}\")\n        return None,None\n    return audio, sr\n\n# Function to augment audio with pitch shifting, time stretching, and adding noise\ndef augment_audio(audio, sr, noise_level=0.005, pitch_factor=0.5, stretch_factor=0.8):\n    try:\n        # Pitch shifting\n        audio_pitched = librosa.effects.pitch_shift(audio, n_steps=pitch_factor, sr=sr)\n        \n        # Time stretching\n        audio_stretched = librosa.effects.time_stretch(audio_pitched, rate=stretch_factor)\n        \n        # Adding random noise\n        noise = np.random.randn(len(audio_stretched)) * noise_level\n        audio_noisy = audio_stretched + noise\n    except Exception as e:\n        print(f\"Error augmenting audio: {e}\")\n        return audio  # Return unmodified audio if error occurs\n\n    return audio_noisy\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T10:40:01.120278Z","iopub.execute_input":"2024-05-31T10:40:01.120796Z","iopub.status.idle":"2024-05-31T10:40:01.357731Z","shell.execute_reply.started":"2024-05-31T10:40:01.120710Z","shell.execute_reply":"2024-05-31T10:40:01.356588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Extraction","metadata":{}},{"cell_type":"code","source":"# Function to extract audio features\ndef extract_features(audio, sr, mfcc_length=130):\n    # Spectral features\n    mfccs = librosa.feature.mfcc(y=audio, sr=sr, n_mfcc=13)  # Mel-frequency cepstral coefficients\n    spectral_contrast = librosa.feature.spectral_contrast(y=audio, sr=sr)  # Spectral contrast\n    chroma_stft = librosa.feature.chroma_stft(y=audio, sr=sr)  # Chroma features\n\n    # Temporal features\n    zero_crossing_rate = librosa.feature.zero_crossing_rate(audio)[0]  # Zero-crossing rate\n    energy = np.sum(np.square(audio))  # Energy\n    energy_entropy = -np.sum(audio**2 * np.log(audio**2 + 1e-10))  # Entropy of energy\n\n    # Collect all features in a dictionary\n    features = {\n        'mfccs': mfccs,\n        'spectral_contrast': spectral_contrast,\n        'chroma_stft': chroma_stft,\n        'zero_crossing_rate': zero_crossing_rate,\n        'energy': energy,\n        'energy_entropy': energy_entropy\n    }\n    return features","metadata":{"execution":{"iopub.status.busy":"2024-05-31T10:40:05.430124Z","iopub.execute_input":"2024-05-31T10:40:05.430648Z","iopub.status.idle":"2024-05-31T10:40:05.441557Z","shell.execute_reply.started":"2024-05-31T10:40:05.430604Z","shell.execute_reply":"2024-05-31T10:40:05.440247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function to batch process audio files","metadata":{}},{"cell_type":"code","source":"def get_metadata_info(full_filename, metadata):\n    # 通过完整的文件名查找元数据\n    info = metadata[metadata['filename'] == full_filename].iloc[0]\n    return info\n\ndef batch_process_audio_files(file_paths, metadata, augment=False, max_workers=4):\n    processed_features = []\n    labels = []\n    remark = []\n    \n    with ThreadPoolExecutor(max_workers=max_workers) as executor:\n        futures = []\n        for file_path in file_paths:\n            full_filename = Path(file_path).parent.name + '/' + Path(file_path).name\n            meta_info = get_metadata_info(full_filename, metadata)\n            futures.append(executor.submit(load_and_normalize_audio, file_path, augment=augment))\n            futures[-1].file_path = file_path  # 将 file_path 附加到 future 对象\n            futures[-1].meta_info = meta_info  # 将 meta_info 附加到 future 对象\n        \n        for future in futures:\n            audio_data = future.result()\n            if audio_data is not None:\n                audio, sr = audio_data\n                features = extract_features(audio, sr)\n                meta_info = future.meta_info  # 获取附加的 meta_info\n                features.update({\n                    'full_label': meta_info['filename'],\n                    'bird_name': meta_info['primary_label'],\n                    'second_labels': meta_info['secondary_labels'],\n                    'type': meta_info['type'],\n                    'latitude': meta_info['latitude'],\n                    'longitude': meta_info['longitude'],\n                    'scientific_name': meta_info['scientific_name'],\n                    'common_name': meta_info['common_name'],\n                    'rating': meta_info['rating']\n                })\n                processed_features.append(features)\n                labels.append(meta_info['primary_label'])\n                remark.append(future.file_path)  # 获取附加的 file_path\n            else:\n                print(f\"Failed to process audio file {future.file_path}\")\n                \n    return processed_features, labels, remark\n\n# Example usage\ntop_level_directory_train = '/kaggle/input/birdclef-2024/train_audio'\nsample_size=50\nsampled_audio_files = sample_ogg_files(top_level_directory_train, sample_size)\n\nprocessed_features, labels, remark = batch_process_audio_files(sampled_audio_files, metadata, augment=True)\nprint(f\"Processed {len(processed_features)} files successfully.\")","metadata":{"execution":{"iopub.status.busy":"2024-05-31T10:40:23.597900Z","iopub.execute_input":"2024-05-31T10:40:23.598323Z","iopub.status.idle":"2024-05-31T10:41:56.575446Z","shell.execute_reply.started":"2024-05-31T10:40:23.598286Z","shell.execute_reply":"2024-05-31T10:41:56.574245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"# 显示处理后的特征和标签\n# print(labels)\n# print(sampled_audio_files)\n# print(remark)\n\n# Spectrograms:\ndef plot_spectrogram(audio, sr, title=\"Spectrogram\"):\n    plt.figure(figsize=(10, 4))\n    D = librosa.amplitude_to_db(np.abs(librosa.stft(audio)), ref=np.max)\n    librosa.display.specshow(D, sr=sr, x_axis='time', y_axis='hz')\n    plt.colorbar(format='%+2.0f dB')\n    plt.title(title)\n    plt.show()\n\ndef plot_waveform(audio, sr, title=\"Audio Waveform\"):\n    plt.figure(figsize=(10, 3))\n    librosa.display.waveshow(audio, sr=sr)\n    plt.title(title)\n    plt.xlabel('Time')\n    plt.ylabel('Amplitude')\n    plt.show()\n\ndef plot_mfccs(mfccs, sr, title=\"MFCC\"):\n    plt.figure(figsize=(10, 4))\n    librosa.display.specshow(mfccs, x_axis='time')\n    plt.colorbar()\n    plt.title(title)\n    plt.tight_layout()\n    plt.show()\n\n# Feature Scatter Plots:\nfeatures_df = pd.DataFrame([{\n    **feat,\n    'mfcc_avg': np.mean(feat['mfccs']),\n    'energy': feat['energy']\n} for feat in processed_features])\n\n# 分组展示鸟类名称，按10个一组\ngroup_size = 20\nbird_names = features_df['bird_name'].unique()\nnum_groups = (len(bird_names) + group_size - 1) // group_size\n\nfor i in range(num_groups):\n    group_bird_names = bird_names[i * group_size:(i + 1) * group_size]\n    group_features_df = features_df[features_df['bird_name'].isin(group_bird_names)]\n\n    plt.figure(figsize=(10, 6))\n    sns.scatterplot(data=group_features_df, x='mfcc_avg', y='energy', hue='bird_name')\n    plt.title('Scatter plot of MFCC Average vs Energy')\n    plt.xlabel('MFCC Average')\n    plt.ylabel('Energy')\n    plt.legend(title='Bird Name', bbox_to_anchor=(1.05, 1), loc='upper left')\n    plt.show()\n\n# 选择一个音频进行展示\naudio_data, sr = load_and_normalize_audio(sampled_audio_files[0]) \nplot_spectrogram(audio_data, sr, \"Example Spectrogram\")\nplot_waveform(audio_data, sr, \"Audio Waveform\")\n\nmfccs = librosa.feature.mfcc(y=audio_data, sr=sr, n_mfcc=13)\nplot_mfccs(mfccs, sr, \"MFCC Heatmap\")\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T10:43:14.816300Z","iopub.execute_input":"2024-05-31T10:43:14.816731Z","iopub.status.idle":"2024-05-31T10:43:18.778986Z","shell.execute_reply.started":"2024-05-31T10:43:14.816683Z","shell.execute_reply":"2024-05-31T10:43:18.777786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Development","metadata":{}},{"cell_type":"markdown","source":"# Baseline Model","metadata":{}},{"cell_type":"code","source":"# 转换为 DataFrame\nfeatures_df = pd.DataFrame(processed_features)\n\n# 提取统计特征\ndef extract_stat_features(row):\n    row['mfccs_mean'] = np.mean(row['mfccs'], axis=1).mean()  # 取平均 MFCCs\n    row['mfccs_var'] = np.var(row['mfccs'], axis=1).mean()  # 取方差 MFCCs\n    row['spectral_contrast_mean'] = np.mean(row['spectral_contrast'])\n    row['chroma_stft_mean'] = np.mean(row['chroma_stft'], axis=1).mean()\n    return row\n\n# 应用统计特征提取\nfeatures_df = features_df.apply(extract_stat_features, axis=1)\n\n# 删除原始的复杂特征列\nfeatures_df.drop(['mfccs', 'spectral_contrast', 'chroma_stft', 'zero_crossing_rate'], axis=1, inplace=True)\n\n# 使用 LabelEncoder 转换鸟类名称为整数标签\nlabel_encoder = LabelEncoder()\nfeatures_df['encoded_labels'] = label_encoder.fit_transform(features_df['bird_name'])\n\n# 选择特征和标签（确保所有特征都是数值型）\nX = features_df.select_dtypes(include=[np.number])  # 选择所有数值型列\n\n# 需要手动添加已编码的标签列如果它不是数值型的\nif 'encoded_labels' in features_df:\n    X['encoded_labels'] = features_df['encoded_labels']\n    \n# 处理缺失值\nimputer = SimpleImputer(strategy='mean')\nX = imputer.fit_transform(X)\n\ny = features_df['encoded_labels']\n\n# 划分训练集和测试集\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n\n# 创建并训练随机森林模型\nrf = RandomForestClassifier(n_estimators=100, random_state=42)\nrf.fit(X_train, y_train)\n\n# 进行预测\ny_pred = rf.predict(X_test)\n\n# 获取随机森林模型的预测概率\nrf_predictions_proba = rf.predict_proba(X_test)\n\n# 评估模型\naccuracy = accuracy_score(y_test, y_pred)\nprint(\"Accuracy of the RandomForest model: {:.2f}%\".format(accuracy * 100))\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:54:52.231460Z","iopub.execute_input":"2024-05-31T11:54:52.231942Z","iopub.status.idle":"2024-05-31T11:54:52.655343Z","shell.execute_reply.started":"2024-05-31T11:54:52.231904Z","shell.execute_reply":"2024-05-31T11:54:52.653977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deep learning Models","metadata":{}},{"cell_type":"code","source":"features_df2 = pd.DataFrame(processed_features)\nfeatures_df2['bird_name'] = labels\n\n# 提取 MFCC 特征并确保所有特征具有相同的形状\nmfcc_features = np.array([librosa.util.fix_length(m, size=130, axis=1) for m in features_df2['mfccs']])\nmfcc_features = mfcc_features[..., np.newaxis]  # 添加一个新的维度以匹配 CNN 输入形状\n# 标签编码\nlabel_encoder2 = LabelEncoder()\nencoded_labels2 = label_encoder2.fit_transform(features_df2['bird_name'])\nencoded_labels2 = to_categorical(encoded_labels2)\n\n# CNNs Model\n\n# 划分训练集和测试集\nX_train_cnn, X_test_cnn, y_train_cnn, y_test_cnn = train_test_split(mfcc_features, encoded_labels2, test_size=0.3, random_state=42)\n\n# 超参数调优的函数\ndef perform_hyperparameter_tuning():\n    # Define the CNN model with Keras Tuner\n    def build_cnn_model(hp):\n        model = Sequential()\n        model.add(Input(shape=(13, 130, 1)))\n        \n        for i in range(hp.Int('num_layers', 2, 4)):\n            model.add(Conv2D(\n                filters=hp.Int(f'filters_{i}', 32, 128, step=32),\n                kernel_size=eval(hp.Choice(f'kernel_size_{i}', ['(3, 3)', '(5, 5)'])),\n                activation='relu',\n                padding='same'  # 使用 'same' 填充以避免尺寸减小过多\n            ))\n            model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))  # 使用 'same' 填充\n            model.add(Dropout(rate=hp.Float(f'dropout_{i}', 0.1, 0.5, step=0.1)))\n        \n        model.add(Flatten())\n        model.add(Dense(\n            units=hp.Int('units', 128, 512, step=128),\n            activation='relu'\n        ))\n        model.add(Dropout(rate=hp.Float('dense_dropout', 0.2, 0.5, step=0.1)))\n        model.add(Dense(len(label_encoder2.classes_), activation='softmax'))\n\n        model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n        return model\n\n    # Hyperparameter tuning\n    tuner = kt.Hyperband(\n        build_cnn_model,\n        objective='val_accuracy',\n        max_epochs=20,\n        factor=3,\n        directory='hyperband',\n        project_name='birdsong'\n    )\n\n    # 需要在每次调用 tuner.search 时创建新的 EarlyStopping 实例\n    tuner.search(X_train_cnn, y_train_cnn, epochs=50, validation_data=(X_test_cnn, y_test_cnn), callbacks=[EarlyStopping(monitor='val_loss', patience=5)])\n\n    # Get the optimal hyperparameters\n    best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]\n\n    print(f\"\"\"\n    The optimal number of layers is {best_hps.get('num_layers')} with the following configuration:\n    \"\"\")\n    for i in range(best_hps.get('num_layers')):\n        print(f\"Layer {i+1}: filters={best_hps.get(f'filters_{i}')}, kernel_size={best_hps.get(f'kernel_size_{i}')}, dropout={best_hps.get(f'dropout_{i}')}\")\n    print(f\"Dense layer: units={best_hps.get('units')}, dropout={best_hps.get('dense_dropout')}\")\n\n    # Build the model with the optimal hyperparameters\n    cnn_model = tuner.hypermodel.build(best_hps)\n\n    # Train the model\n    cnn_history = cnn_model.fit(X_train_cnn, y_train_cnn, epochs=50, batch_size=32, validation_data=(X_test_cnn, y_test_cnn), callbacks=[EarlyStopping(monitor='val_loss', patience=5)])\n\n    return cnn_model, cnn_history\n\n# 调用函数进行超参数调优\ncnn_model, cnn_history = perform_hyperparameter_tuning()\n\n# 获取CNN模型的预测概率\ncnn_predictions_proba = cnn_model.predict(X_test_cnn)\n\n# 评估模型\ntest_loss_cnn, test_acc_cnn = cnn_model.evaluate(X_test_cnn, y_test_cnn)\nprint(f\"CNN Test accuracy: {test_acc_cnn:.2f}\")\n\n# RNN Model\n\n# 超参数调优的函数\ndef perform_rnn_hyperparameter_tuning():\n    # 创建RNN模型\n    def build_rnn_model(hp):\n        model = Sequential()\n        model.add(Input(shape=(13, 130, 1)))\n        model.add(Reshape((13, 130)))\n        \n        for i in range(hp.Int('num_layers', 1, 2)):\n            model.add(LSTM(\n                units=hp.Int(f'units_{i}', 64, 256, step=64),\n                return_sequences=True if i < hp.get('num_layers') - 1 else False\n            ))\n            model.add(Dropout(rate=hp.Float(f'dropout_{i}', 0.1, 0.5, step=0.1)))\n        \n        model.add(Dense(len(label_encoder2.classes_), activation='softmax'))\n\n        model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n        return model\n\n    # Hyperparameter tuning for RNN\n    tuner_rnn = kt.Hyperband(\n        build_rnn_model,\n        objective='val_accuracy',\n        max_epochs=20,\n        factor=3,\n        directory='hyperband',\n        project_name='birdsong_rnn'\n    )\n\n    # 需要在每次调用 tuner.search 时创建新的 EarlyStopping 实例\n    tuner_rnn.search(X_train_cnn, y_train_cnn, epochs=50, validation_data=(X_test_cnn, y_test_cnn), callbacks=[EarlyStopping(monitor='val_loss', patience=5)])\n\n    # Get the optimal hyperparameters for RNN\n    best_hps_rnn = tuner_rnn.get_best_hyperparameters(num_trials=1)[0]\n\n    print(f\"\"\"\n    The optimal number of layers is {best_hps_rnn.get('num_layers')} with the following configuration:\n    \"\"\")\n    for i in range(best_hps_rnn.get('num_layers')):\n        print(f\"Layer {i+1}: units={best_hps_rnn.get(f'units_{i}')}, dropout={best_hps_rnn.get(f'dropout_{i}')}\")\n\n    # Build the RNN model with the optimal hyperparameters\n    rnn_model = tuner_rnn.hypermodel.build(best_hps_rnn)\n\n    # Train the RNN model\n    rnn_history = rnn_model.fit(X_train_cnn, y_train_cnn, epochs=50, batch_size=32, validation_data=(X_test_cnn, y_test_cnn), callbacks=[EarlyStopping(monitor='val_loss', patience=5)])\n\n    return rnn_model, rnn_history\n\n# 调用函数进行超参数调优\nrnn_model, rnn_history = perform_rnn_hyperparameter_tuning()\n\n# 获取RNN模型的预测概率\nrnn_predictions_proba = rnn_model.predict(X_test_cnn)\n\n# 评估模型\ntest_loss_rnn, test_acc_rnn = rnn_model.evaluate(X_test_cnn, y_test_cnn)\nprint(f\"RNN Test accuracy: {test_acc_rnn:.2f}\")\n\n# 检查形状是否一致\nprint(rf_predictions_proba.shape)\nprint(cnn_predictions_proba.shape)\nprint(rnn_predictions_proba.shape)\n\n\n# 确保所有预测概率的形状一致\ndef ensure_same_shape(arr1, arr2):\n    if arr1.shape[1] > arr2.shape[1]:\n        pad_width = arr1.shape[1] - arr2.shape[1]\n        arr2 = np.pad(arr2, ((0, 0), (0, pad_width)), 'constant')\n    elif arr2.shape[1] > arr1.shape[1]:\n        pad_width = arr2.shape[1] - arr1.shape[1]\n        arr1 = np.pad(arr1, ((0, 0), (0, pad_width)), 'constant')\n    return arr1, arr2\n\nrf_predictions_proba, cnn_predictions_proba = ensure_same_shape(rf_predictions_proba, cnn_predictions_proba)\nrf_predictions_proba, rnn_predictions_proba = ensure_same_shape(rf_predictions_proba, rnn_predictions_proba)\ncnn_predictions_proba, rnn_predictions_proba = ensure_same_shape(cnn_predictions_proba, rnn_predictions_proba)\n\n# Ensemble Model\n\n# 将所有模型的预测概率进行平均\nensemble_predictions_proba = (rf_predictions_proba + cnn_predictions_proba + rnn_predictions_proba) / 3\n\n# 选择概率最高的类别作为最终预测结果\nensemble_labels = np.argmax(ensemble_predictions_proba, axis=1) \n\n# 计算集成模型的准确率\nensemble_accuracy = accuracy_score(np.argmax(y_test_cnn, axis=1), ensemble_labels)\nprint(f\"Ensemble model accuracy: {ensemble_accuracy:.2f}\")\n\n\n# 解码预测结果以查看实际鸟类名称\nensemble_predicted_bird_names = label_encoder.inverse_transform(ensemble_labels)\nprint(ensemble_predicted_bird_names[:5])\n\n\n# Visualization of training and validation accuracy and loss of CNN models\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(cnn_history.history['accuracy'], label='Training Accuracy')\nplt.plot(cnn_history.history['val_accuracy'], label='Validation Accuracy')\nplt.title('CNN Training and Validation Accuracy') \nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(cnn_history.history['loss'], label='Training Loss')\nplt.plot(cnn_history.history['val_loss'], label='Validation Loss')\nplt.title('CNN Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()\n\n# Visualize the training and validation accuracy and loss of RNN models\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(rnn_history.history['accuracy'], label='RNN Training Accuracy')\nplt.plot(rnn_history.history['val_accuracy'], label='RNN Validation Accuracy')\nplt.title('RNN Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(rnn_history.history['loss'], label='RNN Training Loss')\nplt.plot(rnn_history.history['val_loss'], label='RNN Validation Loss')\nplt.title('RNN Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T11:55:01.113306Z","iopub.execute_input":"2024-05-31T11:55:01.113734Z","iopub.status.idle":"2024-05-31T11:55:11.975264Z","shell.execute_reply.started":"2024-05-31T11:55:01.113683Z","shell.execute_reply":"2024-05-31T11:55:11.973832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Performance Evaluation","metadata":{}},{"cell_type":"code","source":"\n# 生成混淆矩阵\ncm = confusion_matrix(np.argmax(y_test_cnn, axis=1), ensemble_labels)\n\n# 确保标签和位置数量一致\nclasses = label_encoder.classes_\nif len(classes) > cm.shape[0]:\n    classes = classes[:cm.shape[0]]\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=classes, yticklabels=classes)\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.title('Confusion Matrix')\nplt.xticks(rotation=90)\nplt.yticks(rotation=0)\nplt.show()\n\n# 计算 ROC-AUC 和 Precision-Recall 曲线\n# 对多类别问题进行处理\nn_classes = len(label_encoder.classes_)\n\n# Binarize the labels for ROC and Precision-Recall curve computation\ny_test_binarized = label_binarize(np.argmax(y_test_cnn, axis=1), classes=range(n_classes))\n\n# Compute ROC curve and ROC area for each class\nfpr = dict()\ntpr = dict()\nroc_auc = dict()\nprecision = dict()\nrecall = dict()\npr_auc = dict()\n\nfor i in range(n_classes):\n    if np.sum(y_test_binarized[:, i]) == 0:  # Skip if no true instances for this class\n        continue\n    fpr[i], tpr[i], _ = roc_curve(y_test_binarized[:, i], ensemble_predictions_proba[:, i])\n    roc_auc[i] = auc(fpr[i], tpr[i])\n    precision[i], recall[i], _ = precision_recall_curve(y_test_binarized[:, i], ensemble_predictions_proba[:, i])\n    pr_auc[i] = auc(recall[i], precision[i])\n\n# Plot ROC curves\nplt.figure()\nfor i in range(n_classes):\n    if i in fpr:  # Only plot if fpr exists for this class\n        plt.plot(fpr[i], tpr[i], label=f'Class {label_encoder.classes_[i]} (area = {roc_auc[i]:0.2f})')\nplt.plot([0, 1], [0, 1], 'k--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('ROC Curve')\nplt.legend(loc=\"lower right\")\nplt.show()\n\n# Plot Precision-Recall curves\nplt.figure()\nfor i in range(n_classes):\n    if i in precision:  # Only plot if precision exists for this class\n        plt.plot(recall[i], precision[i], label=f'Class {label_encoder.classes_[i]} (area = {pr_auc[i]:0.2f})')\nplt.xlabel('Recall')\nplt.ylabel('Precision')\nplt.title('Precision-Recall Curve')\nplt.legend(loc=\"lower right\")\nplt.show()\n\n# 错误分析\n# 获取所有实际存在的类别\nactual_labels = np.unique(np.argmax(y_test_cnn, axis=1))\npredicted_labels = np.unique(ensemble_labels)\n\n# 获取这些标签的名称\nactual_classes = label_encoder.inverse_transform(actual_labels)\npredicted_classes = label_encoder.inverse_transform(predicted_labels)\n\n# 确保分类报告的标签和目标名称一致\ncommon_classes = np.union1d(actual_labels, predicted_labels)\ntarget_names = label_encoder.inverse_transform(common_classes)\n\nprint(\"Classification Report:\")\nprint(classification_report(np.argmax(y_test_cnn, axis=1), ensemble_labels, labels=common_classes, target_names=target_names))\n\n# 将预测结果和实际结果结合起来分析\nerror_analysis_df = pd.DataFrame({\n    'True Label': label_encoder.inverse_transform(np.argmax(y_test_cnn, axis=1)),\n    'Predicted Label': label_encoder.inverse_transform(ensemble_labels)\n})\nerrors = error_analysis_df[error_analysis_df['True Label'] != error_analysis_df['Predicted Label']]\nprint(\"Errors:\")\nprint(errors.head())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T12:05:46.931296Z","iopub.execute_input":"2024-05-31T12:05:46.932621Z","iopub.status.idle":"2024-05-31T12:05:49.208433Z","shell.execute_reply.started":"2024-05-31T12:05:46.932577Z","shell.execute_reply":"2024-05-31T12:05:49.207263Z"},"trusted":true},"execution_count":null,"outputs":[]}]}