{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":19596,"databundleVersionId":1292430,"sourceType":"competition"},{"sourceId":25954,"databundleVersionId":2091745,"sourceType":"competition"},{"sourceId":33246,"databundleVersionId":3221581,"sourceType":"competition"},{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":8048860,"sourceType":"datasetVersion","datasetId":4745108}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\nlabels = ['BirdCLEF-2025', 'BirdCLEF-2024-extra', 'BirdCLEF-2021', 'BirdCLEF-2024', 'BirdCLEF-2023', 'BirdCLEF-2020', 'BirdCLEF-2022']\nsizes = [28552, 24255, 62, 13, 10, 1, 0]\n\ncolors = plt.cm.Paired(np.linspace(0, 1, len(sizes)))\n\nplt.figure(figsize=(10, 6))\nbars = plt.barh(labels, sizes, color=colors, edgecolor='none')\nfor bar in bars:\n    plt.text(bar.get_width(), bar.get_y() + bar.get_height() / 2,\n             f'{bar.get_width():,.0f}', va='center', ha='left', fontsize=12, fontweight='bold', color='black')\n\nplt.title('# Birdcalls of 2025', fontsize=16, fontweight='bold')\n\nplt.xticks(fontsize=10)\nplt.yticks(fontsize=10)\n\nplt.grid(False)\n\nfor spine in plt.gca().spines.values():\n    spine.set_visible(False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T20:21:34.157879Z","iopub.execute_input":"2025-03-10T20:21:34.158243Z","iopub.status.idle":"2025-03-10T20:21:34.401399Z","shell.execute_reply.started":"2025-03-10T20:21:34.158215Z","shell.execute_reply":"2025-03-10T20:21:34.400227Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [BirdCLEF 2020-2025 All Training npy Dataset](https://www.kaggle.com/datasets/seshurajup/birdclef-2020-2025-all-training-npy/data)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n! rm -rf /kaggle/working/*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:51.383251Z","iopub.execute_input":"2025-03-10T19:35:51.383613Z","iopub.status.idle":"2025-03-10T19:35:51.563907Z","shell.execute_reply.started":"2025-03-10T19:35:51.383581Z","shell.execute_reply":"2025-03-10T19:35:51.562193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = ['primary_label', 'secondary_labels', 'record_name', 'source']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:51.56716Z","iopub.execute_input":"2025-03-10T19:35:51.56752Z","iopub.status.idle":"2025-03-10T19:35:51.574557Z","shell.execute_reply.started":"2025-03-10T19:35:51.567489Z","shell.execute_reply":"2025-03-10T19:35:51.573122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/birdclef-2025/train.csv')\ntrain\n\ntrain['record'] = [filename.split('/')[1] for filename in train.filename]\ntrain['record_name'] = [record.split('.')[0] for record in train.record]\n\ntrain['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train['secondary_labels']]\ntrain['source'] = 'bc25'\ntrain[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:51.576329Z","iopub.execute_input":"2025-03-10T19:35:51.576687Z","iopub.status.idle":"2025-03-10T19:35:51.977183Z","shell.execute_reply.started":"2025-03-10T19:35:51.576657Z","shell.execute_reply":"2025-03-10T19:35:51.976074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_20 = pd.read_csv('/kaggle/input/birdsong-recognition/train.csv')\ntrain_20\n\ntrain_20['record_name'] = [record.split('.')[0] for record in train_20.filename]\ntrain_20['primary_label'] = train_20['ebird_code']\ntrain_20['source'] = 'bc20'\ntrain_20 = train_20[~train_20.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_20 = train_20[train_20.primary_label.isin(train.primary_label.unique())].reset_index(drop=True)\ntrain_20[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:51.978199Z","iopub.execute_input":"2025-03-10T19:35:51.9785Z","iopub.status.idle":"2025-03-10T19:35:52.410191Z","shell.execute_reply.started":"2025-03-10T19:35:51.978473Z","shell.execute_reply":"2025-03-10T19:35:52.409004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = train_20[['species', 'ebird_code']].drop_duplicates().sort_values('species').reset_index(drop=True)\ndf\n\nspecies2code = {species : code for species, code in zip(df.species, df.ebird_code)}\nspecies2code\n\ndef process_secondary_labels(secondary_labels, species2code):\n    secondary_labels = eval(secondary_labels)\n    labels = []\n    for secondary in secondary_labels:\n        label = species2code.get(secondary.split('_')[1], '')\n        if label != '':\n            labels.append(label)\n    return labels\n\ntrain_20['secondary_labels'] = [process_secondary_labels(secondary_labels, species2code) for secondary_labels in train_20['secondary_labels']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:52.411367Z","iopub.execute_input":"2025-03-10T19:35:52.411694Z","iopub.status.idle":"2025-03-10T19:35:52.423323Z","shell.execute_reply.started":"2025-03-10T19:35:52.411663Z","shell.execute_reply":"2025-03-10T19:35:52.42199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_21 = pd.read_csv('/kaggle/input/birdclef-2021/train_metadata.csv')\ntrain_21\ntrain_21['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_21['secondary_labels']]\ntrain_21['record_name'] = [record.split('.')[0] for record in train_21.filename]\ntrain_21['source'] = 'bc21'\ntrain_21 = train_21[~train_21.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_21 = train_21[train_21.primary_label.isin(train.primary_label.unique())].reset_index(drop=True)\ntrain_21[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:52.424181Z","iopub.execute_input":"2025-03-10T19:35:52.424524Z","iopub.status.idle":"2025-03-10T19:35:53.314985Z","shell.execute_reply.started":"2025-03-10T19:35:52.424483Z","shell.execute_reply":"2025-03-10T19:35:53.313986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_22 = pd.read_csv('/kaggle/input/birdclef-2022/train_metadata.csv')\ntrain_22['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_22['secondary_labels']]\ntrain_22['record_name'] = [record.split('/')[1].split('.')[0] for record in train_22.filename]\ntrain_22['source'] = 'bc22'\ntrain_22 = train_22[~train_22.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_22 = train_22[train_22.primary_label.isin(train.primary_label.unique())].reset_index(drop=True)\ntrain_22[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:53.31588Z","iopub.execute_input":"2025-03-10T19:35:53.316141Z","iopub.status.idle":"2025-03-10T19:35:53.517407Z","shell.execute_reply.started":"2025-03-10T19:35:53.316119Z","shell.execute_reply":"2025-03-10T19:35:53.516026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_23 = pd.read_csv('/kaggle/input/birdclef-2023/train_metadata.csv')\ntrain_23['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_23['secondary_labels']]\ntrain_23['record_name'] = [record.split('/')[1].split('.')[0] for record in train_23.filename]\ntrain_23['source'] = 'bc23'\ntrain_23 = train_23[~train_23.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_23 = train_23[train_23.primary_label.isin(train.primary_label.unique())].reset_index(drop=True)\ntrain_23[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:53.520769Z","iopub.execute_input":"2025-03-10T19:35:53.521461Z","iopub.status.idle":"2025-03-10T19:35:53.761344Z","shell.execute_reply.started":"2025-03-10T19:35:53.521424Z","shell.execute_reply":"2025-03-10T19:35:53.760207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_24 = pd.read_csv('/kaggle/input/birdclef-2024/train_metadata.csv')\ntrain_24['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_24['secondary_labels']]\ntrain_24['record_name'] = [record.split('/')[1].split('.')[0] for record in train_24.filename]\ntrain_24['source'] = 'bc24'\ntrain_24 = train_24[~train_24.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_24 = train_24[train_24.primary_label.isin(train.primary_label.unique())].reset_index(drop=True)\ntrain_24[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:53.763045Z","iopub.execute_input":"2025-03-10T19:35:53.763468Z","iopub.status.idle":"2025-03-10T19:35:54.070954Z","shell.execute_reply.started":"2025-03-10T19:35:53.763421Z","shell.execute_reply":"2025-03-10T19:35:54.069564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from glob import glob\nfile_paths = glob(\"/kaggle/input/birdclef2024-additional-mp3/additional_audio\" + \"/*/*\")\nlen(file_paths), file_paths[:5]\ntrain_additional = pd.DataFrame({\n    'filename' : ['/'.join(filepath.split('/')[-2:]) for filepath in file_paths],\n    'species' : [filepath.split('/')[-2] for filepath in file_paths],\n    'record' : [filepath.split('/')[-1] for filepath in file_paths],\n    'filepath': file_paths,\n})\ntrain_additional['source'] = 'bc00'\ntrain_additional['record_name'] = [record.split('.')[0] for record in train_additional.record]\ntrain_additional['primary_label'] = train_additional['species']\ntrain_additional['secondary_labels'] = [[] for _ in train_additional['primary_label']]\ntrain_additional = train_additional[~train_additional.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_additional[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.07179Z","iopub.execute_input":"2025-03-10T19:35:54.072182Z","iopub.status.idle":"2025-03-10T19:35:54.507341Z","shell.execute_reply.started":"2025-03-10T19:35:54.072153Z","shell.execute_reply":"2025-03-10T19:35:54.506296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train = pd.concat([train_20[cols], \n                       train_21[cols], \n                       train_22[cols], \n                       train_23[cols],\n                       train_24[cols],\n                       train[cols],\n                       train_additional[cols]\n                       ]).reset_index(drop=True)\nall_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.508494Z","iopub.execute_input":"2025-03-10T19:35:54.508908Z","iopub.status.idle":"2025-03-10T19:35:54.545946Z","shell.execute_reply.started":"2025-03-10T19:35:54.508863Z","shell.execute_reply":"2025-03-10T19:35:54.544838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train = all_train.sort_values(['primary_label', 'source'], ascending=False).reset_index(drop=True)\nall_train = all_train.drop_duplicates('record_name')\nall_train['rank'] = all_train.groupby('primary_label').source.rank(method='first', ascending=False)\nall_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.547298Z","iopub.execute_input":"2025-03-10T19:35:54.54773Z","iopub.status.idle":"2025-03-10T19:35:54.637208Z","shell.execute_reply.started":"2025-03-10T19:35:54.547686Z","shell.execute_reply":"2025-03-10T19:35:54.635956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train.source.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.638396Z","iopub.execute_input":"2025-03-10T19:35:54.638889Z","iopub.status.idle":"2025-03-10T19:35:54.650894Z","shell.execute_reply.started":"2025-03-10T19:35:54.638847Z","shell.execute_reply":"2025-03-10T19:35:54.649693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train.to_csv('/kaggle/working/all_train.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.652091Z","iopub.execute_input":"2025-03-10T19:35:54.652463Z","iopub.status.idle":"2025-03-10T19:35:54.798214Z","shell.execute_reply.started":"2025-03-10T19:35:54.652432Z","shell.execute_reply":"2025-03-10T19:35:54.796925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\nfrom tqdm import tqdm\nimport librosa\nsr = 32000\n\ndef load_audio(record_name, primary_label, source):\n    if source == 'bc20':\n        pathname = Path('/kaggle/input/birdsong-recognition/train_audio/') / primary_label / (record_name + '.mp3')\n    elif source == 'bc21':\n        pathname = Path('/kaggle/input/birdclef-2021/train_short_audio/') / primary_label / (record_name + '.ogg')\n    elif source == 'bc22':\n        pathname = Path('/kaggle/input/birdclef-2022/train_audio/') / primary_label / (record_name + '.ogg')\n    elif source == 'bc23':\n        pathname = Path('/kaggle/input/birdclef-2023/train_audio/') / primary_label / (record_name + '.ogg')\n    elif source == 'bc24':\n        pathname = Path('/kaggle/input/birdclef-2024/train_audio/') / primary_label / (record_name + '.ogg')\n    audio = librosa.load(pathname, sr=32000)[0].astype(np.float32)\n    return audio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.799331Z","iopub.execute_input":"2025-03-10T19:35:54.799746Z","iopub.status.idle":"2025-03-10T19:35:54.807006Z","shell.execute_reply.started":"2025-03-10T19:35:54.799705Z","shell.execute_reply":"2025-03-10T19:35:54.805724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nlengths = []\nfor record_name, primary_label, source in zip(tqdm(all_train.record_name), all_train.primary_label, all_train.source):\n    if source in ['bc20', 'bc21', 'bc22', 'bc23', 'bc24']:\n        audio = load_audio(record_name, primary_label, source)\n        lengths.append(len(audio))\n        save_path = Path('/kaggle/working/') / primary_label\n        os.makedirs(save_path, exist_ok=True)\n        np.save(save_path / ('first10_' + record_name), audio[: 10 * sr])\n        np.save(save_path / ('last10_' + record_name), audio[-10 * sr : ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-10T19:35:54.808073Z","iopub.execute_input":"2025-03-10T19:35:54.808405Z","iopub.status.idle":"2025-03-10T19:36:00.614238Z","shell.execute_reply.started":"2025-03-10T19:35:54.808377Z","shell.execute_reply":"2025-03-10T19:36:00.612923Z"}},"outputs":[],"execution_count":null}]}