{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":11361501,"sourceType":"datasetVersion","datasetId":7110971}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"bc7cc291-8917-45d4-9d97-9769c82dab9e","cell_type":"code","source":"import os\nimport math\nimport time\nimport librosa\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport torch\nimport torchvision.transforms as transforms\nimport warnings\nimport zipfile\nimport shutil\nimport json\nimport subprocess\n\nwarnings.filterwarnings(\"ignore\")\n\n# 安装并升级 Kaggle API\n!pip install --upgrade kaggle\n\n# 配置 Kaggle API\nkaggle_json_path = '/kaggle/input/kaggle/kaggle.json'\nif not os.path.exists(kaggle_json_path):\n    raise FileNotFoundError(\"请将 kaggle.json 上传到 /kaggle/input/kaggle/ 或修改路径\")\n!mkdir -p ~/.kaggle\n!cp {kaggle_json_path} ~/.kaggle/\n!chmod 600 ~/.kaggle/kaggle.json\n\n# 配置类\nclass Config:\n    DEBUG_MODE = False\n    OUTPUT_DIR = '/kaggle/working'\n    DATA_ROOT = '/kaggle/input/birdclef-2025'\n    FS = 32000\n    N_FFT = 1024\n    HOP_LENGTH = 512\n    N_MELS = 128\n    FMIN = 50\n    FMAX = 14000\n    TARGET_DURATION = 5.0\n    TARGET_SHAPE = (224, 224)\n    N_MAX = 50 if DEBUG_MODE else None\n\nconfig = Config()\n\nclass CFG:\n    seed = 42\n    debug = False\n    apex = False\n    print_freq = 100\n    num_workers = 2\n    OUTPUT_DIR = '/kaggle/working'\n    DATA_ROOT = '/kaggle/input/birdclef-2025'\n    model_name = 'vit_b_16'\n    pretrained = True\n    in_channels = 3\n    LOAD_DATA = True\n    FS = 32000\n    TARGET_DURATION = 5.0\n    TARGET_SHAPE = (224, 224)\n    N_FFT = 1024\n    HOP_LENGTH = 512\n    N_MELS = 128\n    FMIN = 50\n    FMAX = 14000\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    epochs = 10\n    batch_size = 16\n    criterion = 'BCEWithLogitsLoss'\n    n_fold = 5\n    selected_folds = [0, 1, 2, 3, 4]\n    optimizer = 'AdamW'\n    lr = 1e-4\n    weight_decay = 1e-5\n    scheduler = 'CosineAnnealingLR'\n    min_lr = 1e-6\n    T_max = epochs\n    aug_prob = 0.5\n    mixup_alpha = 0.5\n\n    def update_debug_settings(self):\n        if self.debug:\n            self.epochs = 2\n            self.selected_folds = [0]\n\ncfg = CFG()\n\n# 调试信息\nprint(f\"Debug mode: {'ON' if config.DEBUG_MODE else 'OFF'}\")\nprint(f\"Max samples to process: {config.N_MAX if config.N_MAX is not None else 'ALL'}\")\n\n# 加载数据\nprint(\"Loading taxonomy data...\")\ntaxonomy_df = pd.read_csv(f'{config.DATA_ROOT}/taxonomy.csv')\nspecies_class_map = dict(zip(taxonomy_df['primary_label'], taxonomy_df['class_name']))\n\nprint(\"Loading training metadata...\")\ntrain_df = pd.read_csv(f'{config.DATA_ROOT}/train.csv')\nlabel_list = sorted(train_df['primary_label'].unique())\nlabel_id_list = list(range(len(label_list)))\nlabel2id = dict(zip(label_list, label_id_list))\nid2label = dict(zip(label_id_list, label_list))\n\nprint(f'Found {len(label_list)} unique species')\nworking_df = train_df[['primary_label', 'rating', 'filename']].copy()\nworking_df['target'] = working_df.primary_label.map(label2id)\nworking_df['filepath'] = config.DATA_ROOT + '/train_audio/' + working_df.filename\nworking_df['samplename'] = working_df.filename.map(lambda x: x.split('/')[0] + '-' + x.split('/')[-1].split('.')[0])\nworking_df['class'] = working_df.primary_label.map(lambda x: species_class_map.get(x, 'Unknown'))\ntotal_samples = min(len(working_df), config.N_MAX or len(working_df))\nprint(f'Total samples to process: {total_samples} out of {len(working_df)} available')\nprint(f'Samples by class:')\nprint(working_df['class'].value_counts())\n\n# Mel 频谱图生成函数\ndef audio2melspec(audio_data, cfg):\n    if np.isnan(audio_data).any():\n        mean_signal = np.nanmean(audio_data)\n        audio_data = np.nan_to_num(audio_data, nan=mean_signal)\n    mel_spec = librosa.feature.melspectrogram(\n        y=audio_data, sr=cfg.FS, n_fft=cfg.N_FFT, hop_length=cfg.HOP_LENGTH,\n        n_mels=cfg.N_MELS, fmin=cfg.FMIN, fmax=cfg.FMAX, power=2.0\n    )\n    mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)\n    mel_spec_norm = (mel_spec_db - mel_spec_db.min()) / (mel_spec_db.max() - mel_spec_db.min() + 1e-8)\n    mel_spec_tensor = torch.from_numpy(mel_spec_norm).float().unsqueeze(0).repeat(3, 1, 1)\n    vit_transforms = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    return vit_transforms(mel_spec_tensor).numpy()\n\ndef process_audio_file(audio_path, cfg):\n    try:\n        audio_data, _ = librosa.load(audio_path, sr=cfg.FS)\n        target_samples = int(cfg.TARGET_DURATION * cfg.FS)\n        if len(audio_data) < target_samples:\n            n_copy = math.ceil(target_samples / len(audio_data))\n            audio_data = np.concatenate([audio_data] * n_copy) if n_copy > 1 else audio_data\n        start_idx = max(0, int(len(audio_data) / 2 - target_samples / 2))\n        end_idx = min(len(audio_data), start_idx + target_samples)\n        center_audio = audio_data[start_idx:end_idx]\n        if len(center_audio) < target_samples:\n            center_audio = np.pad(center_audio, (0, target_samples - len(center_audio)), mode='constant')\n        return audio2melspec(center_audio, cfg).astype(np.float32)\n    except Exception as e:\n        print(f\"Error processing {audio_path}: {e}\")\n        return None\n\n# 分批生成与压缩\ndef generate_and_zip_spectrograms(df, cfg, batch_size=50, max_zip_size_mb=500):\n    print(\"开始分批生成并压缩 Mel 频谱图...\")\n    start_time = time.time()\n    output_dir = os.path.join(cfg.OUTPUT_DIR, 'spectrograms_batches')\n    os.makedirs(output_dir, exist_ok=True)\n    \n    total_batches = (len(df) + batch_size - 1) // batch_size\n    print(f\"总批次: {total_batches}\")\n    \n    for batch_idx in range(total_batches):\n        total, used, free = shutil.disk_usage(cfg.OUTPUT_DIR)\n        free_gb = free / (1024 ** 3)\n        print(f\"剩余空间: {free_gb:.2f} GB\")\n        if free_gb < 1:\n            print(\"磁盘空间不足，停止处理！\")\n            break\n            \n        batch_start = batch_idx * batch_size\n        batch_end = min((batch_idx + 1) * batch_size, len(df))\n        batch_df = df.iloc[batch_start:batch_end]\n        all_bird_data = {}\n        all_labels = {}\n        \n        for i, row in tqdm(batch_df.iterrows(), total=len(batch_df), desc=f\"批次 {batch_idx+1}/{total_batches}\"):\n            samplename = row['samplename']\n            filepath = row['filepath']\n            label = row['target']\n            mel_spec = process_audio_file(filepath, cfg)\n            if mel_spec is not None:\n                all_bird_data[samplename] = mel_spec\n                all_labels[samplename] = label\n        \n        if not all_bird_data:\n            continue\n        \n        npz_path = os.path.join(output_dir, f'batch_{batch_idx}.npz')\n        batch_keys = list(all_bird_data.keys())\n        batch_spectrograms = np.array(list(all_bird_data.values()))\n        batch_labels = np.array([all_labels[key] for key in batch_keys], dtype=np.int64)\n        np.savez(npz_path, spectrograms=batch_spectrograms, keys=batch_keys, labels=batch_labels)\n        \n        zip_path = os.path.join(cfg.OUTPUT_DIR, f'batch_{batch_idx}.zip')\n        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zip_file:\n            zip_file.write(npz_path, os.path.basename(npz_path))\n        \n        zip_size_mb = os.path.getsize(zip_path) / (1024 * 1024)\n        print(f\"批次 {batch_idx+1} 完成，ZIP 大小: {zip_size_mb:.2f} MB\")\n        \n        total, used, free = shutil.disk_usage(cfg.OUTPUT_DIR)\n        free_gb = free / (1024 ** 3)\n        if free_gb < 0.5:\n            print(f\"警告：剩余空间仅 {free_gb:.2f} GB，建议停止处理！\")\n            break\n        \n        if zip_size_mb > max_zip_size_mb:\n            print(f\"警告：批次 {batch_idx+1} 的 ZIP 文件 ({zip_size_mb:.2f} MB) 超过 {max_zip_size_mb} MB，建议减小 batch_size。\")\n        \n        os.remove(npz_path)\n    \n    end_time = time.time()\n    print(f\"所有批次处理完成，总耗时 {end_time - start_time:.2f} 秒\")\n\nimport subprocess\n\ndef upload_to_kaggle(output_dir):\n    try:\n        print(\"正在上传到 Kaggle 数据集...\")\n        # 检查数据集是否已存在\n        result = subprocess.run(\n            \"kaggle datasets list -m | grep birdclef-2025-spectrograms\",\n            shell=True, capture_output=True, text=True\n        )\n        if \"birdclef-2025-spectrograms\" in result.stdout:\n            print(\"数据集已存在，正在更新版本...\")\n            cmd = f\"kaggle datasets version -p {output_dir} --dir-mode zip -m 'Updated spectrograms batch'\"\n        else:\n            print(\"创建新数据集...\")\n            cmd = f\"kaggle datasets create -p {output_dir} --dir-mode zip\"\n        \n        result = subprocess.run(\n            cmd, shell=True, check=True, capture_output=True, text=True\n        )\n        print(\"上传完成！\")\n        print(\"命令输出:\", result.stdout)\n    except subprocess.CalledProcessError as e:\n        print(f\"上传失败: {e}\")\n        print(f\"错误输出: {e.stderr if e.stderr else '无详细错误信息'}\")\n        print(f\"请检查 Kaggle API 配置、用户名、数据集是否存在，或网络连接\")\n    except Exception as e:\n        print(f\"上传过程中发生未知错误: {e}\")\n\n# 执行\nbatch_size = 20\nmax_zip_size_mb = 500\ngenerate_and_zip_spectrograms(working_df, cfg, batch_size=batch_size, max_zip_size_mb=max_zip_size_mb)\n\n# # 创建 Kaggle 数据集元数据\n# metadata = {\n#     \"title\": \"birdclef-2025-spectrograms\",\n#     \"id\": \"hongkongbaptist/birdclef-2025-spectrograms\",  # 请确保用户名正确\n#     \"licenses\": [{\"name\": \"CC0-1.0\"}]\n# }\n# print(\"请确保 'hongkongbaptist' 是您的 Kaggle 用户名，否则修改 metadata['id']\")\n# with open(os.path.join(cfg.OUTPUT_DIR, 'dataset-metadata.json'), 'w') as f:\n#     json.dump(metadata, f)\n\n# upload_to_kaggle(cfg.OUTPUT_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T02:42:34.886376Z","iopub.execute_input":"2025-04-11T02:42:34.886735Z"}},"outputs":[],"execution_count":null},{"id":"a210b27d-c2c6-4402-8565-3bd95ac8905d","cell_type":"code","source":"# import os\n# import json\n# import subprocess\n\n# # Step 1: Update Kaggle API\n# !pip install kaggle --upgrade\n# !kaggle --version\n\n# # Step 2: Verify API credentials\n# !ls -l ~/.kaggle/kaggle.json\n# !kaggle datasets list\n\n# # Step 3: Regenerate dataset-metadata.json\n# metadata = {\n#     \"title\": \"birdclef-2025-spectrograms\",\n#     \"id\": \"hongkongbaptist/birdclef-2025-spectrograms\",\n#     \"licenses\": [{\"name\": \"CC0-1.0\"}]\n# }\n# with open(os.path.join(cfg.OUTPUT_DIR, 'dataset-metadata.json'), 'w') as f:\n#     json.dump(metadata, f, indent=2)\n\n# # Verify metadata\n# !cat /kaggle/working/dataset-metadata.json\n\n# # Step 4: Reduce number of files to test\n# !mkdir -p /kaggle/working/temp\n# !mv /kaggle/working/batch_*.zip /kaggle/working/temp/\n# !mv /kaggle/working/temp/batch_0.zip /kaggle/working/\n# !mv /kaggle/working/temp/batch_1.zip /kaggle/working/\n# !ls -lh /kaggle/working\n\n# # Step 5: Retry upload\n# def upload_to_kaggle(output_dir):\n#     try:\n#         print(\"正在上传到 Kaggle 数据集...\")\n#         result = subprocess.run(\n#             [\"kaggle\", \"datasets\", \"list\", \"-m\"],\n#             capture_output=True, text=True\n#         )\n#         dataset_id = \"hongkongbaptist/birdclef-2025-spectrograms\"\n#         if \"birdclef-2025-spectrograms\" in result.stdout:\n#             print(\"数据集已存在，正在更新版本...\")\n#             cmd = [\n#                 \"kaggle\", \"datasets\", \"version\",\n#                 \"-p\", output_dir,\n#                 \"--dir-mode\", \"zip\",\n#                 \"-m\", \"Updated spectrograms batch\"\n#             ]\n#         else:\n#             print(\"创建新数据集...\")\n#             cmd = [\n#                 \"kaggle\", \"datasets\", \"create\",\n#                 \"-p\", output_dir,\n#                 \"--dir-mode\", \"zip\"\n#             ]\n        \n#         result = subprocess.run(cmd, check=True, capture_output=True, text=True)\n#         print(\"上传完成！\")\n#         print(\"命令输出:\", result.stdout)\n#     except subprocess.CalledProcessError as e:\n#         print(f\"上传失败: {e}\")\n#         print(f\"错误输出: {e.stderr}\")\n#     except Exception as e:\n#         print(f\"上传过程中发生未知错误: {e}\")\n\n# upload_to_kaggle(cfg.OUTPUT_DIR)\n\n# # Step 6: If the above fails, try a minimal test dataset\n# !mkdir /kaggle/working/test-dataset\n# !echo \"test content\" > /kaggle/working/test-dataset/test.txt\n# test_metadata = {\n#     \"title\": \"test-birdclef-2025\",\n#     \"id\": \"hongkongbaptist/test-birdclef-2025\",\n#     \"licenses\": [{\"name\": \"CC0-1.0\"}]\n# }\n# with open('/kaggle/working/test-dataset/dataset-metadata.json', 'w') as f:\n#     json.dump(test_metadata, f, indent=2)\n\n# !kaggle datasets create -p /kaggle/working/test-dataset --dir-mode zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T08:37:07.767826Z","iopub.execute_input":"2025-04-11T08:37:07.768504Z","iopub.status.idle":"2025-04-11T08:50:02.730298Z","shell.execute_reply.started":"2025-04-11T08:37:07.768470Z","shell.execute_reply":"2025-04-11T08:50:02.675284Z"}},"outputs":[],"execution_count":null},{"id":"92bc6a6e-a4b9-49a4-9fd5-7e933dc8d476","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}