{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11351027,"sourceType":"datasetVersion","datasetId":7102743}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# BirdCLEF_Demucs_Demo.ipynb\n\n# 安装必要的库\n!pip install -q demucs librosa matplotlib soundfile torchaudio\n\nimport os\nimport numpy as np\nimport librosa\nimport librosa.display\nimport matplotlib.pyplot as plt\nimport torch\nimport torchaudio\nimport soundfile as sf\nfrom IPython.display import Audio, display\nfrom pathlib import Path\n\n# 确保我们使用GPU（如果可用）\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"使用设备: {device}\")\n\n# 加载Demucs模型\nprint(\"加载Demucs模型...\")\nfrom demucs.pretrained import get_model\nfrom demucs.apply import apply_model\n\n# 使用htdemucs模型，它在分离人声和乐器方面表现非常好\nmodel = get_model(\"htdemucs\")\nmodel.to(device)\nprint(f\"模型加载完成: {model.sources}\")  # 通常包括['drums', 'bass', 'other', 'vocals']\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T06:59:42.539418Z","iopub.execute_input":"2025-04-10T06:59:42.540002Z","iopub.status.idle":"2025-04-10T07:01:13.089771Z","shell.execute_reply.started":"2025-04-10T06:59:42.539984Z","shell.execute_reply":"2025-04-10T07:01:13.089046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 修改visualize_audio函数，使用英文显示\ndef visualize_audio(audio_data, sr, title=\"Audio Waveform and Spectrogram\"):\n    \"\"\"\n    Display audio waveform and spectrogram\n    显示音频波形和频谱图\n    \"\"\"\n    plt.figure(figsize=(14, 6))\n    \n    # Waveform (波形图)\n    plt.subplot(2, 1, 1)\n    librosa.display.waveshow(audio_data, sr=sr)\n    plt.title(f\"{title} - Waveform\")  # 波形图标题\n    plt.xlabel(\"Time (s)\")  # 时间轴标签\n    plt.ylabel(\"Amplitude\")  # 振幅轴标签\n    \n    # Spectrogram (频谱图)\n    plt.subplot(2, 1, 2)\n    D = librosa.amplitude_to_db(np.abs(librosa.stft(audio_data)), ref=np.max)\n    librosa.display.specshow(D, sr=sr, x_axis='time', y_axis='log')\n    plt.colorbar(format='%+2.0f dB')\n    plt.title(f\"{title} - Spectrogram\")  # 频谱图标题\n    plt.tight_layout()\n    plt.show()\n\n# 修改separate_voice函数，使用英文显示\ndef separate_voice(audio_path, output_dir=\"separated_audio\"):\n    \"\"\"\n    Use Demucs to separate vocals from other sounds in audio\n    使用Demucs分离音频中的人声和其他声音\n    \n    Parameters (参数):\n    audio_path: Path to audio file (音频文件路径)\n    output_dir: Directory to save separated audio (分离后的音频保存目录)\n    \n    Returns (返回):\n    Dictionary of separated audio sources (分离后的音频字典)\n    Sample rate (采样率)\n    \"\"\"\n    # Create output directory (创建输出目录)\n    os.makedirs(output_dir, exist_ok=True)\n    \n    # Load audio (加载音频)\n    print(f\"Loading audio file: {audio_path}\")  # 加载音频文件\n    try:\n        # Use librosa to load audio, keeping original sample rate (使用librosa加载音频，保持原采样率)\n        audio_data, sr = librosa.load(audio_path, sr=None, mono=True)\n        \n        # Print audio information (打印音频信息)\n        print(f\"Audio info: Sample rate={sr}Hz, Duration={len(audio_data)/sr:.2f}s\")\n        \n        # Visualize original audio (可视化原始音频)\n        visualize_audio(audio_data, sr, title=\"Original Audio\")  # 原始音频\n        display(Audio(audio_data, rate=sr))\n        \n        # Fix: Convert mono to stereo by duplicating the channel (通过复制通道将单声道转换为立体声)\n        # Demucs expects input shape [batch, channels, time] with channels=2\n        stereo_audio = np.stack([audio_data, audio_data])  # 复制单声道到两个通道\n        audio_tensor = torch.tensor(stereo_audio, device=device).unsqueeze(0)  # 形状变为 [1, 2, time]\n        \n        print(f\"Converted audio shape: {audio_tensor.shape}\")  # 转换后的音频形状\n        \n        # Separate audio (分离音频)\n        print(\"Separating audio with Demucs...\")  # 使用Demucs分离音频\n        with torch.no_grad():\n            sources = apply_model(model, audio_tensor, device=device)[0]\n        \n        # Extract each part from sources (从sources中提取各个部分)\n        # 4 sources: drums, bass, other, vocals\n        sources_dict = {}\n        for source, source_audio in zip(model.sources, sources):\n            # Take first channel of stereo output (取立体声输出的第一个通道)\n            source_audio_mono = source_audio[0].cpu().numpy()\n            sources_dict[source] = source_audio_mono\n            \n            # Save separated audio (保存分离的音频)\n            output_path = os.path.join(output_dir, f\"{Path(audio_path).stem}_{source}.wav\")\n            sf.write(output_path, source_audio_mono, sr)\n            print(f\"Saved separated audio: {output_path}\")  # 已保存分离的音频\n        \n        # Create no-vocals version (combining all sources except vocals)\n        # 创建无人声版本 (合并除vocals外的所有源)\n        no_vocals = sum([sources_dict[source] for source in model.sources if source != 'vocals'])\n        sources_dict['no_vocals'] = no_vocals\n        \n        # Save no-vocals version (保存无人声版本)\n        output_path = os.path.join(output_dir, f\"{Path(audio_path).stem}_no_vocals.wav\")\n        sf.write(output_path, no_vocals, sr)\n        print(f\"Saved no-vocals audio: {output_path}\")  # 已保存无人声音频\n        \n        # Visualize separated audio (可视化分离后的音频)\n        vocals = sources_dict['vocals']\n        visualize_audio(vocals, sr, title=\"Separated Vocals\")  # 分离出的人声\n        display(Audio(vocals, rate=sr))\n        \n        visualize_audio(no_vocals, sr, title=\"Audio without Vocals\")  # 无人声音频\n        display(Audio(no_vocals, rate=sr))\n        \n        return sources_dict, sr\n        \n    except Exception as e:\n        print(f\"Error processing audio: {e}\")  # 处理音频时出错\n        import traceback\n        traceback.print_exc()\n        return None, None\n\n# 修改compare_spectrograms函数，使用英文显示\ndef compare_spectrograms(audio_original, audio_no_vocals, sr, title=\"Original vs No-Vocals Audio\"):\n    \"\"\"\n    Compare mel spectrograms of two audio samples\n    比较两个音频的梅尔频谱图\n    \"\"\"\n    # Calculate mel spectrograms (计算梅尔频谱图)\n    mel_spec_orig = librosa.feature.melspectrogram(\n        y=audio_original, sr=sr, n_fft=1024, hop_length=512, \n        n_mels=128, fmin=50, fmax=14000\n    )\n    mel_spec_no_vocals = librosa.feature.melspectrogram(\n        y=audio_no_vocals, sr=sr, n_fft=1024, hop_length=512, \n        n_mels=128, fmin=50, fmax=14000\n    )\n    \n    # Convert to decibels (转换为分贝)\n    mel_spec_db_orig = librosa.power_to_db(mel_spec_orig, ref=np.max)\n    mel_spec_db_no_vocals = librosa.power_to_db(mel_spec_no_vocals, ref=np.max)\n    \n    # Visualize comparison (可视化比较)\n    plt.figure(figsize=(14, 8))\n    \n    plt.subplot(2, 1, 1)\n    librosa.display.specshow(\n        mel_spec_db_orig, sr=sr, hop_length=512, \n        x_axis='time', y_axis='mel', fmin=50, fmax=14000\n    )\n    plt.colorbar(format='%+2.0f dB')\n    plt.title('Original Audio Mel Spectrogram')  # 原始音频的梅尔频谱图\n    \n    plt.subplot(2, 1, 2)\n    librosa.display.specshow(\n        mel_spec_db_no_vocals, sr=sr, hop_length=512, \n        x_axis='time', y_axis='mel', fmin=50, fmax=14000\n    )\n    plt.colorbar(format='%+2.0f dB')\n    plt.title('No-Vocals Audio Mel Spectrogram')  # 无人声音频的梅尔频谱图\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # Calculate spectrogram difference (vocals portion)\n    # 计算频谱图差异（人声部分）\n    diff = mel_spec_db_orig - mel_spec_db_no_vocals\n    \n    plt.figure(figsize=(10, 4))\n    librosa.display.specshow(\n        diff, sr=sr, hop_length=512, \n        x_axis='time', y_axis='mel', fmin=50, fmax=14000\n    )\n    plt.colorbar(format='%+2.0f dB')\n    plt.title('Spectrogram Difference (Original - No Vocals)')  # 频谱图差异 (原始 - 无人声)\n    plt.tight_layout()\n    plt.show()\n    \n    return mel_spec_db_orig, mel_spec_db_no_vocals, diff\n\n# 修改主测试函数\ndef test_demucs_on_samples(sample_paths):\n    \"\"\"\n    Test Demucs separation on multiple samples\n    在多个样本上测试Demucs分离效果\n    \"\"\"\n    results = []\n    \n    for audio_path in sample_paths:\n        print(f\"\\nProcessing sample: {audio_path}\")  # 处理样本\n        try:\n            # Separate vocals (分离人声)\n            sources_dict, sr = separate_voice(audio_path)\n            \n            if sources_dict and sr:\n                # Compare spectrograms (比较频谱图)\n                original_audio = librosa.load(audio_path, sr=sr)[0]\n                no_vocals_audio = sources_dict['no_vocals']\n                \n                mel_orig, mel_no_vocals, diff = compare_spectrograms(original_audio, no_vocals_audio, sr)\n                \n                results.append({\n                    'path': audio_path,\n                    'sources': sources_dict,\n                    'sr': sr,\n                    'mel_original': mel_orig,\n                    'mel_no_vocals': mel_no_vocals,\n                    'mel_diff': diff\n                })\n            \n        except Exception as e:\n            print(f\"Error processing {audio_path}: {e}\")  # 处理时出错\n    \n    return results\n\n# 主程序更新\nsample_paths = [\n    # Replace with actual file paths (替换为实际的文件路径)\n    # Recommended to use Fabio A. Sarria-S recordings (推荐选择Fabio A. Sarria-S的录音)\n    '/kaggle/input/birdclef-2025/train_audio/1139490/CSA36385.ogg',\n    '/kaggle/input/birdclef-2025/train_audio/1139490/CSA36389.ogg',\n]\n\n# Run test (运行测试)\nprint(\"Testing Demucs voice separation...\")  # 测试Demucs人声分离\nresults = test_demucs_on_samples(sample_paths[:2])  # 只测试前两个样本以节省时间\n\nprint(\"\\nSeparation complete! Check the output directory for the separated audio files.\")\n# 分离完成! 查看输出目录中的文件以获取分离后的音频。","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T07:18:24.395209Z","iopub.execute_input":"2025-04-10T07:18:24.395631Z","iopub.status.idle":"2025-04-10T07:19:10.979886Z","shell.execute_reply.started":"2025-04-10T07:18:24.395609Z","shell.execute_reply":"2025-04-10T07:19:10.979220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 从人声数据文件中加载样本\nprint(\"\\n从已知含人声的音频文件中测试Demucs\")\n\nimport pickle\nimport random\n# 修正提取相对路径的方法\ndef extract_relative_path(full_path):\n    \"\"\"从完整路径中提取出train_audio/之后的相对路径\"\"\"\n    if 'train_audio/' in full_path:\n        return full_path.split('train_audio/')[-1]\n    return full_path\n# 加载已知包含人声的文件列表\ntry:\n    with open(voice_files_path, 'r') as f:\n        voice_files = [line.strip() for line in f.readlines()]\n    print(f\"成功加载了 {len(voice_files)} 个已知包含人声的文件\")\n    \n    # 修正路径 - 提取train_audio/后面的部分作为相对路径\n    voice_files = [extract_relative_path(path) for path in voice_files]\n    \n    # 加载人声时间戳数据（可选，用于更详细的分析）\n    try:\n        with open(voice_data_path, 'rb') as f:\n            voice_timestamps = pickle.load(f)\n        print(\"成功加载人声时间戳数据\")\n        has_timestamps = True\n    except Exception as e:\n        print(f\"加载人声时间戳数据时出错: {e}\")\n        voice_timestamps = {}\n        has_timestamps = False\n    \n    # 加载训练数据\n    train_csv_path = '/kaggle/input/birdclef-2025/train.csv'\n    train_df = pd.read_csv(train_csv_path)\n    \n    # 为每个评分级别随机选择含人声的样本\n    random.seed(42)  # 固定随机种子\n    \n    # 找出文件名在voice_files中的样本\n    voice_files_set = set(voice_files)\n    voice_samples_df = train_df[train_df['filename'].apply(lambda x: x in voice_files_set)]\n    print(f\"找到 {len(voice_samples_df)} 个匹配的样本\")\n    \n    # 按评分分组，并为每个评分选择样本\n    rating_samples = {}\n    for rating in range(6):  # 0 to 5\n        rating_df = voice_samples_df[voice_samples_df['rating'] == rating]\n        \n        if len(rating_df) > 0:\n            # 如果评分组有多个样本，随机选择最多3个\n            if len(rating_df) > 3:\n                samples = rating_df.sample(3, random_state=42)\n            else:\n                samples = rating_df\n                \n            paths = [f'/kaggle/input/birdclef-2025/train_audio/{filename}' for filename in samples['filename']]\n            rating_samples[rating] = paths\n            print(f\"评分 {rating}: 选择了 {len(paths)} 个含人声的样本\")\n        else:\n            rating_samples[rating] = []\n            print(f\"评分 {rating}: 未找到含人声的样本\")\n    \n    # 如果某些评分级别没有含人声的样本，从所有含人声样本中随机选择\n    if any(len(paths) == 0 for rating, paths in rating_samples.items()):\n        # 随机打乱样本\n        all_voice_paths = [f'/kaggle/input/birdclef-2025/train_audio/{filename}' for filename in voice_samples_df['filename']]\n        random.shuffle(all_voice_paths)\n        \n        # 为空的评分组分配样本\n        for rating in rating_samples:\n            if len(rating_samples[rating]) == 0 and all_voice_paths:\n                rating_samples[rating] = [all_voice_paths.pop()]\n                print(f\"评分 {rating}: 从所有含人声样本中随机分配了1个样本\")\n    \n    # 按评分测试样本\n    print(\"\\n开始按评分级别测试含人声的样本...\")\n    for rating, paths in rating_samples.items():\n        if not paths:\n            print(f\"\\n跳过评分 {rating}: 没有可用样本\")\n            continue\n            \n        print(f\"\\n### 测试评分为 {rating} 的含人声样本 ###\")\n        \n        # 为每个评分仅测试一个样本以节省时间\n        sample_path = paths[0]\n        try:\n            print(f\"处理样本: {sample_path}\")\n            \n            # 如果有时间戳数据，显示人声出现的时间段\n            if has_timestamps and sample_path in voice_timestamps:\n                timestamps = voice_timestamps[sample_path]\n                print(f\"人声出现在以下时间段（秒）:\")\n                for ts in timestamps[:5]:  # 最多显示5个时间戳\n                    print(f\"  {ts['start']:.2f} - {ts['end']:.2f}\")\n                if len(timestamps) > 5:\n                    print(f\"  ... 共 {len(timestamps)} 个人声片段\")\n            \n            # 分离音频\n            sources_dict, sr = separate_voice(sample_path)\n            \n            if sources_dict and sr:\n                # 比较频谱图\n                original_audio = librosa.load(sample_path, sr=sr)[0]\n                no_vocals_audio = sources_dict['no_vocals']\n                \n                # 计算梅尔频谱图\n                mel_orig, mel_no_vocals, diff = compare_spectrograms(original_audio, no_vocals_audio, sr)\n                \n                # 显示分离的听觉效果\n                print(\"原始音频:\")\n                display(Audio(original_audio, rate=sr))\n                \n                print(\"分离出的人声:\")\n                display(Audio(sources_dict['vocals'], rate=sr))\n                \n                print(\"无人声音频（鸟叫声）:\")\n                display(Audio(no_vocals_audio, rate=sr))\n                \n        except Exception as e:\n            print(f\"处理评分为 {rating} 的样本时出错: {e}\")\n            import traceback\n            traceback.print_exc()\n\n    # 如果要测试具体的样本（不按评分分组）\n    print(\"\\n测试几个特定的人声样本...\")\n    \n    # 随机从可用样本中选择不超过5个\n    test_samples = random.sample(all_voice_paths, min(5, len(all_voice_paths)))\n    \n    for i, sample_path in enumerate(test_samples):\n        print(f\"\\n### 测试样本 {i+1}/{len(test_samples)} ###\")\n        try:\n            print(f\"处理样本: {sample_path}\")\n            \n            # 分离音频\n            sources_dict, sr = separate_voice(sample_path)\n            \n            if sources_dict and sr:\n                # 比较频谱图\n                original_audio = librosa.load(sample_path, sr=sr)[0]\n                no_vocals_audio = sources_dict['no_vocals']\n                vocals_audio = sources_dict['vocals']\n                \n                # 计算梅尔频谱图\n                mel_orig, mel_no_vocals, diff = compare_spectrograms(original_audio, no_vocals_audio, sr)\n                \n                # 显示分离的听觉效果\n                print(\"原始音频:\")\n                display(Audio(original_audio, rate=sr))\n                \n                print(\"分离出的人声:\")\n                display(Audio(vocals_audio, rate=sr))\n                \n                print(\"无人声音频（鸟叫声）:\")\n                display(Audio(no_vocals_audio, rate=sr))\n        except Exception as e:\n            print(f\"处理样本时出错: {e}\")\n            import traceback\n            traceback.print_exc()\n    \nexcept Exception as e:\n    print(f\"处理人声数据文件时出错: {e}\")\n    import traceback\n    traceback.print_exc()\n\nprint(\"\\n基于人声数据的测试完成!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-10T08:14:52.286574Z","iopub.execute_input":"2025-04-10T08:14:52.287136Z","iopub.status.idle":"2025-04-10T08:16:07.365804Z","shell.execute_reply.started":"2025-04-10T08:14:52.287112Z","shell.execute_reply":"2025-04-10T08:16:07.365117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 在实际项目中集成Demucs\nprint(\"\\n如何在你的BirdCLEF项目中集成Demucs:\")\nprint(\"\"\"\n# 1. 创建一个过滤人声的函数\ndef filter_vocals_with_demucs(audio_path, sr=32000):\n    # 加载音频\n    audio_tensor, orig_sr = torchaudio.load(audio_path)\n    if orig_sr != sr:\n        audio_tensor = torchaudio.functional.resample(audio_tensor, orig_sr, sr)\n    \n    # 确保音频是单声道\n    if audio_tensor.shape[0] > 1:\n        audio_tensor = torch.mean(audio_tensor, dim=0, keepdim=True)\n    \n    # 分离音频\n    with torch.no_grad():\n        sources = apply_model(model, audio_tensor.to(device), device=device)[0]\n    \n    # 创建无人声版本 (合并除vocals外的所有源)\n    sources_dict = {}\n    for i, source in enumerate(model.sources):\n        sources_dict[source] = sources[i].cpu()\n    \n    # 合并非人声部分\n    no_vocals = sum([sources_dict[source] for source in model.sources if source != 'vocals'])\n    \n    return no_vocals.numpy()\n\n# 2. 在Dataset类中使用这个函数\n# 在BirdCLEFDataset类的__getitem__方法中:\n\ndef __getitem__(self, idx):\n    # ... 现有代码 ...\n    \n    # 如果需要处理人声\n    if self.remove_vocals and os.path.exists(row['filepath']):\n        # 使用Demucs去除人声\n        audio_no_vocals = filter_vocals_with_demucs(row['filepath'])\n        \n        # 使用无人声音频生成频谱图\n        spec = self._audio_to_melspec(audio_no_vocals)\n    else:\n        # 使用原始方法处理音频\n        spec = process_audio_file(row['filepath'], self.cfg)\n    \n    # ... 剩余代码 ...\n\"\"\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}