{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":19596,"databundleVersionId":1292430,"sourceType":"competition"},{"sourceId":25954,"databundleVersionId":2091745,"sourceType":"competition"},{"sourceId":33246,"databundleVersionId":3221581,"sourceType":"competition"},{"sourceId":44224,"databundleVersionId":5188730,"sourceType":"competition"},{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":91844,"databundleVersionId":11361821,"sourceType":"competition"},{"sourceId":8048860,"sourceType":"datasetVersion","datasetId":4745108}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Credits goes to [@cpmpml](https://www.kaggle.com/cpmpml), its a copy of his notebook for 2025 [BirdCLEF 2024 3rd solution](https://github.com/jfpuget/birdclef-2024/)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\nlabels = ['BirdCLEF-2025', 'BirdCLEF-2024-extra', 'BirdCLEF-2021', 'BirdCLEF-2024', 'BirdCLEF-2023', 'BirdCLEF-2020', 'BirdCLEF-2022']\nsizes = [28552, 83, 62, 13, 10, 1, 0]\n\ncolors = plt.cm.Paired(np.linspace(0, 1, len(sizes)))\n\nplt.figure(figsize=(10, 6))\nbars = plt.barh(labels, sizes, color=colors, edgecolor='none')\nfor bar in bars:\n    plt.text(bar.get_width(), bar.get_y() + bar.get_height() / 2,\n             f'{bar.get_width():,.0f}', va='center', ha='left', fontsize=12, fontweight='bold', color='black')\n\nplt.title('# BirdCLEF of 2025', fontsize=16, fontweight='bold')\n\nplt.xticks(fontsize=10)\nplt.yticks(fontsize=10)\n\nplt.grid(False)\n\nfor spine in plt.gca().spines.values():\n    spine.set_visible(False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:05:50.069411Z","iopub.execute_input":"2025-03-22T04:05:50.069673Z","iopub.status.idle":"2025-03-22T04:05:50.227683Z","shell.execute_reply.started":"2025-03-22T04:05:50.069654Z","shell.execute_reply":"2025-03-22T04:05:50.226667Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [BirdCLEF 2020-2025 All Training npy Dataset](https://www.kaggle.com/datasets/seshurajup/birdclef-2020-2025-all-training-npy/data)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n! rm -rf /kaggle/working/*","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:06:12.882352Z","iopub.execute_input":"2025-03-22T04:06:12.882623Z","iopub.status.idle":"2025-03-22T04:06:13.004644Z","shell.execute_reply.started":"2025-03-22T04:06:12.882603Z","shell.execute_reply":"2025-03-22T04:06:13.003645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = ['primary_label', 'secondary_labels', 'record_name', 'source']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T03:59:38.170512Z","iopub.execute_input":"2025-03-22T03:59:38.171129Z","iopub.status.idle":"2025-03-22T03:59:38.175195Z","shell.execute_reply.started":"2025-03-22T03:59:38.171101Z","shell.execute_reply":"2025-03-22T03:59:38.174086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/birdclef-2025/train.csv')\ntrain\n\ntrain['record'] = [filename.split('/')[1] for filename in train.filename]\ntrain['record_name'] = [record.split('.')[0] for record in train.record]\n\ntrain['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train['secondary_labels']]\ntrain['source'] = 'bc25'\ntrain[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T03:59:38.800037Z","iopub.execute_input":"2025-03-22T03:59:38.800327Z","iopub.status.idle":"2025-03-22T03:59:39.097601Z","shell.execute_reply.started":"2025-03-22T03:59:38.800301Z","shell.execute_reply":"2025-03-22T03:59:39.096809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unique_primary_labels = train.primary_label.unique()\nlen(unique_primary_labels), str(unique_primary_labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:01:59.298959Z","iopub.execute_input":"2025-03-22T04:01:59.299227Z","iopub.status.idle":"2025-03-22T04:01:59.305196Z","shell.execute_reply.started":"2025-03-22T04:01:59.299205Z","shell.execute_reply":"2025-03-22T04:01:59.304465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_20 = pd.read_csv('/kaggle/input/birdsong-recognition/train.csv')\ntrain_20\n\ntrain_20['record_name'] = [record.split('.')[0] for record in train_20.filename]\ntrain_20['primary_label'] = train_20['ebird_code']\ntrain_20['source'] = 'bc20'\ntrain_20 = train_20[~train_20.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_20 = train_20[train_20.primary_label.isin(unique_primary_labels)].reset_index(drop=True)\ntrain_20[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:02:51.588635Z","iopub.execute_input":"2025-03-22T04:02:51.588943Z","iopub.status.idle":"2025-03-22T04:02:51.834651Z","shell.execute_reply.started":"2025-03-22T04:02:51.588918Z","shell.execute_reply":"2025-03-22T04:02:51.833728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = train_20[['species', 'ebird_code']].drop_duplicates().sort_values('species').reset_index(drop=True)\ndf\n\nspecies2code = {species : code for species, code in zip(df.species, df.ebird_code)}\nspecies2code\n\ndef process_secondary_labels(secondary_labels, species2code):\n    secondary_labels = eval(secondary_labels)\n    labels = []\n    for secondary in secondary_labels:\n        label = species2code.get(secondary.split('_')[1], '')\n        if label != '':\n            labels.append(label)\n    return labels\n\ntrain_20['secondary_labels'] = [process_secondary_labels(secondary_labels, species2code) for secondary_labels in train_20['secondary_labels']]\ntrain_20 = train_20[~train_20.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_20 = train_20[train_20.primary_label.isin(unique_primary_labels)].reset_index(drop=True)\ntrain_20","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:02:53.830345Z","iopub.execute_input":"2025-03-22T04:02:53.830595Z","iopub.status.idle":"2025-03-22T04:02:53.858127Z","shell.execute_reply.started":"2025-03-22T04:02:53.830574Z","shell.execute_reply":"2025-03-22T04:02:53.857346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_21 = pd.read_csv('/kaggle/input/birdclef-2021/train_metadata.csv')\ntrain_21\ntrain_21['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_21['secondary_labels']]\ntrain_21['record_name'] = [record.split('.')[0] for record in train_21.filename]\ntrain_21['source'] = 'bc21'\ntrain_21 = train_21[~train_21.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_21 = train_21[train_21.primary_label.isin(unique_primary_labels)].reset_index(drop=True)\ntrain_21[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:03:06.241562Z","iopub.execute_input":"2025-03-22T04:03:06.241894Z","iopub.status.idle":"2025-03-22T04:03:06.975505Z","shell.execute_reply.started":"2025-03-22T04:03:06.241872Z","shell.execute_reply":"2025-03-22T04:03:06.974835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_22 = pd.read_csv('/kaggle/input/birdclef-2022/train_metadata.csv')\ntrain_22['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_22['secondary_labels']]\ntrain_22['record_name'] = [record.split('/')[1].split('.')[0] for record in train_22.filename]\ntrain_22['source'] = 'bc22'\ntrain_22 = train_22[~train_22.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_22 = train_22[train_22.primary_label.isin(unique_primary_labels)].reset_index(drop=True)\ntrain_22[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:03:14.095728Z","iopub.execute_input":"2025-03-22T04:03:14.096037Z","iopub.status.idle":"2025-03-22T04:03:14.257967Z","shell.execute_reply.started":"2025-03-22T04:03:14.096016Z","shell.execute_reply":"2025-03-22T04:03:14.257277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_23 = pd.read_csv('/kaggle/input/birdclef-2023/train_metadata.csv')\ntrain_23['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_23['secondary_labels']]\ntrain_23['record_name'] = [record.split('/')[1].split('.')[0] for record in train_23.filename]\ntrain_23['source'] = 'bc23'\ntrain_23 = train_23[~train_23.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_23 = train_23[train_23.primary_label.isin(train.primary_label.unique())].reset_index(drop=True)\ntrain_23[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:03:44.465371Z","iopub.execute_input":"2025-03-22T04:03:44.465628Z","iopub.status.idle":"2025-03-22T04:03:44.669619Z","shell.execute_reply.started":"2025-03-22T04:03:44.465611Z","shell.execute_reply":"2025-03-22T04:03:44.669046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_24 = pd.read_csv('/kaggle/input/birdclef-2024/train_metadata.csv')\ntrain_24['secondary_labels'] = [eval(secondary_labels) for secondary_labels in train_24['secondary_labels']]\ntrain_24['record_name'] = [record.split('/')[1].split('.')[0] for record in train_24.filename]\ntrain_24['source'] = 'bc24'\ntrain_24 = train_24[~train_24.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_24 = train_24[train_24.primary_label.isin(unique_primary_labels)].reset_index(drop=True)\ntrain_24[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:03:21.624348Z","iopub.execute_input":"2025-03-22T04:03:21.624623Z","iopub.status.idle":"2025-03-22T04:03:21.909816Z","shell.execute_reply.started":"2025-03-22T04:03:21.624600Z","shell.execute_reply":"2025-03-22T04:03:21.909063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from glob import glob\nfile_paths = glob(\"/kaggle/input/birdclef2024-additional-mp3/additional_audio\" + \"/*/*\")\nlen(file_paths), file_paths[:5]\ntrain_additional = pd.DataFrame({\n    'filename' : ['/'.join(filepath.split('/')[-2:]) for filepath in file_paths],\n    'species' : [filepath.split('/')[-2] for filepath in file_paths],\n    'record' : [filepath.split('/')[-1] for filepath in file_paths],\n    'filepath': file_paths,\n})\ntrain_additional['source'] = 'bc00'\ntrain_additional['record_name'] = [record.split('.')[0] for record in train_additional.record]\ntrain_additional['primary_label'] = train_additional['species']\ntrain_additional['secondary_labels'] = [[] for _ in train_additional['primary_label']]\ntrain_additional = train_additional[~train_additional.record_name.isin(train.record_name.unique())].reset_index(drop=True)\ntrain_additional = train_additional[train_additional.primary_label.isin(unique_primary_labels)].reset_index(drop=True)\ntrain_additional[cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:04:48.361652Z","iopub.execute_input":"2025-03-22T04:04:48.361965Z","iopub.status.idle":"2025-03-22T04:04:48.655899Z","shell.execute_reply.started":"2025-03-22T04:04:48.361943Z","shell.execute_reply":"2025-03-22T04:04:48.655014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train = pd.concat([train_20[cols], \n                       train_21[cols], \n                       train_22[cols], \n                       train_23[cols],\n                       train_24[cols],\n                       train[cols],\n                       train_additional[cols]\n                       ]).reset_index(drop=True)\nall_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:04:52.069396Z","iopub.execute_input":"2025-03-22T04:04:52.069670Z","iopub.status.idle":"2025-03-22T04:04:52.087634Z","shell.execute_reply.started":"2025-03-22T04:04:52.069649Z","shell.execute_reply":"2025-03-22T04:04:52.086850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train = all_train.sort_values(['primary_label', 'source'], ascending=False).reset_index(drop=True)\nall_train = all_train.drop_duplicates('record_name')\nall_train['rank'] = all_train.groupby('primary_label').source.rank(method='first', ascending=False)\nall_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:04:54.944273Z","iopub.execute_input":"2025-03-22T04:04:54.944531Z","iopub.status.idle":"2025-03-22T04:04:54.985370Z","shell.execute_reply.started":"2025-03-22T04:04:54.944511Z","shell.execute_reply":"2025-03-22T04:04:54.984467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train.source.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:04:57.391918Z","iopub.execute_input":"2025-03-22T04:04:57.392259Z","iopub.status.idle":"2025-03-22T04:04:57.399487Z","shell.execute_reply.started":"2025-03-22T04:04:57.392229Z","shell.execute_reply":"2025-03-22T04:04:57.398688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! mkdir /datasets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:05:00.887190Z","iopub.execute_input":"2025-03-22T04:05:00.887458Z","iopub.status.idle":"2025-03-22T04:05:01.012817Z","shell.execute_reply.started":"2025-03-22T04:05:00.887436Z","shell.execute_reply":"2025-03-22T04:05:01.011323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train.to_csv('/datasets/all_train.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:05:05.703416Z","iopub.execute_input":"2025-03-22T04:05:05.703698Z","iopub.status.idle":"2025-03-22T04:05:05.756261Z","shell.execute_reply.started":"2025-03-22T04:05:05.703675Z","shell.execute_reply":"2025-03-22T04:05:05.755489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\nfrom tqdm import tqdm\nimport librosa\nsr = 32000\n\ndef load_audio(record_name, primary_label, source):\n    if source == 'bc00':\n        pathname = Path('/kaggle/input/birdclef2024-additional-mp3/additional_audio/') / primary_label / (record_name + '.mp3')\n    elif source == 'bc20':\n        pathname = Path('/kaggle/input/birdsong-recognition/train_audio/') / primary_label / (record_name + '.mp3')\n    elif source == 'bc21':\n        pathname = Path('/kaggle/input/birdclef-2021/train_short_audio/') / primary_label / (record_name + '.ogg')\n    elif source == 'bc22':\n        pathname = Path('/kaggle/input/birdclef-2022/train_audio/') / primary_label / (record_name + '.ogg')\n    elif source == 'bc23':\n        pathname = Path('/kaggle/input/birdclef-2023/train_audio/') / primary_label / (record_name + '.ogg')\n    elif source == 'bc24':\n        pathname = Path('/kaggle/input/birdclef-2024/train_audio/') / primary_label / (record_name + '.ogg')\n    audio = librosa.load(pathname, sr=32000)[0].astype(np.float32)\n    return audio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T03:42:40.766415Z","iopub.execute_input":"2025-03-22T03:42:40.766715Z","iopub.status.idle":"2025-03-22T03:42:40.800842Z","shell.execute_reply.started":"2025-03-22T03:42:40.766682Z","shell.execute_reply":"2025-03-22T03:42:40.799900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# lengths = []\n# for record_name, primary_label, source in zip(tqdm(all_train.record_name), all_train.primary_label, all_train.source):\n#     if source in ['bc00', 'bc20', 'bc21', 'bc22', 'bc23', 'bc24']:\n#         audio = load_audio(record_name, primary_label, source)\n#         lengths.append(len(audio))\n#         save_path = Path('/datasets/') / primary_label\n#         os.makedirs(save_path, exist_ok=True)\n#         np.save(save_path / ('first_10_' + record_name), audio[: 10 * sr])\n#         np.save(save_path / ('last_10_' + record_name), audio[-10 * sr : ])\n#         print(record_name, primary_label, source)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-22T04:05:20.225000Z","iopub.execute_input":"2025-03-22T04:05:20.225286Z","iopub.status.idle":"2025-03-22T04:05:20.228128Z","shell.execute_reply.started":"2025-03-22T04:05:20.225265Z","shell.execute_reply":"2025-03-22T04:05:20.227484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}