{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":44224,"databundleVersionId":5188730},{"sourceType":"modelInstanceVersion","sourceId":3836,"databundleVersionId":5146208,"modelInstanceId":2739,"modelId":319}],"dockerImageVersionId":30407,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, we'll use a pre-trained machine learning model to generate a submission to the [BirdClef2023 competition](https://www.kaggle.com/c/birdclef-2023). The goal of the competition is to identify Eastern African bird species by sound.","metadata":{}},{"cell_type":"markdown","source":"## Step 1: Imports","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\nimport tensorflow_io as tfio\n\nimport pandas as pd\nimport numpy as np\nimport librosa\nimport glob\n\nimport csv\nimport io\n\nfrom IPython.display import Audio","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-18T13:16:14.432965Z","iopub.execute_input":"2024-05-18T13:16:14.433764Z","iopub.status.idle":"2024-05-18T13:16:22.049835Z","shell.execute_reply.started":"2024-05-18T13:16:14.433722Z","shell.execute_reply":"2024-05-18T13:16:22.048894Z"},"trusted":true},"outputs":[],"execution_count":1},{"cell_type":"markdown","source":"## Step 2: Explore the training data\n\nWe'll start by loading a couple of training examples and using the IPython.display.Audio module to play them!","metadata":{}},{"cell_type":"code","source":"# Load a sample audio files from two different species\naudio_abe, sr_abe = librosa.load(\"/kaggle/input/birdclef-2023/train_audio/abethr1/XC128013.ogg\")\naudio_abh, sr_abh = librosa.load(\"/kaggle/input/birdclef-2023/train_audio/abhori1/XC127317.ogg\")","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:22.051663Z","iopub.execute_input":"2024-05-18T13:16:22.052298Z","iopub.status.idle":"2024-05-18T13:16:32.318607Z","shell.execute_reply.started":"2024-05-18T13:16:22.052267Z","shell.execute_reply":"2024-05-18T13:16:32.317205Z"},"trusted":true},"outputs":[],"execution_count":2},{"cell_type":"code","source":"# Play the audio\nAudio(data=audio_abe, rate=sr_abe)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:32.32036Z","iopub.execute_input":"2024-05-18T13:16:32.321191Z","iopub.status.idle":"2024-05-18T13:16:32.388917Z","shell.execute_reply.started":"2024-05-18T13:16:32.321154Z","shell.execute_reply":"2024-05-18T13:16:32.38763Z"},"trusted":true},"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"","text/html":"\n \n "},"metadata":{}}],"execution_count":3},{"cell_type":"code","source":"# Play the audio\nAudio(data=audio_abh, rate=sr_abh)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:32.392508Z","iopub.execute_input":"2024-05-18T13:16:32.3929Z","iopub.status.idle":"2024-05-18T13:16:32.452843Z","shell.execute_reply.started":"2024-05-18T13:16:32.392839Z","shell.execute_reply":"2024-05-18T13:16:32.451437Z"},"trusted":true},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"","text/html":"\n \n "},"metadata":{}}],"execution_count":4},{"cell_type":"markdown","source":"## Step 3: Match the model's output with the bird species in the competition\n\nThe competition includes 264 classes of birds, 261 of which exist in this model. We'll set up a way to map the model's output logits to our competition.","metadata":{}},{"cell_type":"code","source":"model = hub.load('https://kaggle.com/models/google/bird-vocalization-classifier/frameworks/tensorFlow2/variations/bird-vocalization-classifier/versions/1')\nlabels_path = hub.resolve('https://kaggle.com/models/google/bird-vocalization-classifier/frameworks/tensorFlow2/variations/bird-vocalization-classifier/versions/1') + \"/assets/label.csv\"","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:32.454326Z","iopub.execute_input":"2024-05-18T13:16:32.45466Z","iopub.status.idle":"2024-05-18T13:16:44.018677Z","shell.execute_reply.started":"2024-05-18T13:16:32.454627Z","shell.execute_reply":"2024-05-18T13:16:44.017465Z"},"trusted":true},"outputs":[],"execution_count":5},{"cell_type":"code","source":"# Find the name of the class with the top score when mean-aggregated across frames.\ndef class_names_from_csv(class_map_csv_text):\n \"\"\"Returns list of class names corresponding to score vector.\"\"\"\n with open(labels_path) as csv_file:\n csv_reader = csv.reader(csv_file, delimiter=',')\n class_names = [mid for mid, desc in csv_reader]\n return class_names[1:]\n\n## note that the bird classifier classifies a much larger set of birds than the\n## competition, so we need to load the model's set of class names or else our \n## indices will be off.\nclasses = class_names_from_csv(labels_path)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:44.020127Z","iopub.execute_input":"2024-05-18T13:16:44.020433Z","iopub.status.idle":"2024-05-18T13:16:44.042139Z","shell.execute_reply.started":"2024-05-18T13:16:44.020404Z","shell.execute_reply":"2024-05-18T13:16:44.041278Z"},"trusted":true},"outputs":[],"execution_count":6},{"cell_type":"code","source":"train_metadata = pd.read_csv(\"/kaggle/input/birdclef-2023/train_metadata.csv\")\ntrain_metadata.head()\ncompetition_classes = sorted(train_metadata.primary_label.unique())\n\nforced_defaults = 0\ncompetition_class_map = []\nfor c in competition_classes:\n try:\n i = classes.index(c)\n competition_class_map.append(i)\n except:\n competition_class_map.append(0)\n forced_defaults += 1\n \n## this is the count of classes not supported by our pretrained model\n## you could choose to simply not predict these, set a default as above,\n## or create your own model using the pretrained model as a base.\nforced_defaults","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:44.043332Z","iopub.execute_input":"2024-05-18T13:16:44.043632Z","iopub.status.idle":"2024-05-18T13:16:44.218693Z","shell.execute_reply.started":"2024-05-18T13:16:44.043604Z","shell.execute_reply":"2024-05-18T13:16:44.217434Z"},"trusted":true},"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"3"},"metadata":{}}],"execution_count":7},{"cell_type":"markdown","source":"## Step 4: Preprocess the data\n\nThe following functions are one way to load the audio provided and break it up into the five-second samples with a sample rate of 32,000 required by the competition.","metadata":{}},{"cell_type":"code","source":"def frame_audio(\n audio_array: np.ndarray,\n window_size_s: float = 5.0,\n hop_size_s: float = 5.0,\n sample_rate = 32000,\n ) -> np.ndarray:\n \n \"\"\"Helper function for framing audio for inference.\"\"\"\n \"\"\" using tf.signal \"\"\"\n if window_size_s is None or window_size_s < 0:\n return audio_array[np.newaxis, :]\n frame_length = int(window_size_s * sample_rate)\n hop_length = int(hop_size_s * sample_rate)\n framed_audio = tf.signal.frame(audio_array, frame_length, hop_length, pad_end=True)\n return framed_audio\n\ndef ensure_sample_rate(waveform, original_sample_rate,\n desired_sample_rate=32000):\n \"\"\"Resample waveform if required.\"\"\"\n if original_sample_rate != desired_sample_rate:\n waveform = tfio.audio.resample(waveform, original_sample_rate, desired_sample_rate)\n return desired_sample_rate, waveform","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:44.220449Z","iopub.execute_input":"2024-05-18T13:16:44.22095Z","iopub.status.idle":"2024-05-18T13:16:44.230448Z","shell.execute_reply.started":"2024-05-18T13:16:44.220909Z","shell.execute_reply":"2024-05-18T13:16:44.229366Z"},"trusted":true},"outputs":[],"execution_count":8},{"cell_type":"markdown","source":"Below we load one training sample - use the Audio function to listen to the samples inside the notebook!","metadata":{}},{"cell_type":"code","source":"audio, sample_rate = librosa.load(\"/kaggle/input/birdclef-2023/train_audio/afghor1/XC156639.ogg\")\nsample_rate, wav_data = ensure_sample_rate(audio, sample_rate)\nAudio(wav_data, rate=sample_rate)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:44.231938Z","iopub.execute_input":"2024-05-18T13:16:44.232364Z","iopub.status.idle":"2024-05-18T13:16:44.867841Z","shell.execute_reply.started":"2024-05-18T13:16:44.23232Z","shell.execute_reply":"2024-05-18T13:16:44.866664Z"},"trusted":true},"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"","text/html":"\n \n "},"metadata":{}}],"execution_count":9},{"cell_type":"markdown","source":"## Step 5: Make predictions\n\nEach test sample is cut into 5-second chunks. We use the pretrained model to return probabilities for all 10k birds included in the model, then pull out the classes used in this competition to create a final submission row. Note that we are NOT doing anything special to handle the 3 missing classes; those will need fine-tuning / transfer learning, which will be handled in a separate notebook.","metadata":{}},{"cell_type":"code","source":"fixed_tm = frame_audio(wav_data)\nlogits, embeddings = model.infer_tf(fixed_tm[:1])\nprobabilities = tf.nn.softmax(logits)\nargmax = np.argmax(probabilities)\nprint(f\"The audio is from the class {classes[argmax]} (element:{argmax} in the label.csv file), with probability of {probabilities[0][argmax]}\")","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:44.871836Z","iopub.execute_input":"2024-05-18T13:16:44.872272Z","iopub.status.idle":"2024-05-18T13:16:57.885966Z","shell.execute_reply.started":"2024-05-18T13:16:44.872231Z","shell.execute_reply":"2024-05-18T13:16:57.884918Z"},"trusted":true},"outputs":[{"name":"stdout","text":"The audio is from the class afghor1 (element:46 in the label.csv file), with probability of 0.5588808655738831\n","output_type":"stream"}],"execution_count":10},{"cell_type":"code","source":"def predict_for_sample(filename, sample_submission, frame_limit_secs=None):\n file_id = filename.split(\".ogg\")[0].split(\"/\")[-1]\n \n audio, sample_rate = librosa.load(filename)\n sample_rate, wav_data = ensure_sample_rate(audio, sample_rate)\n \n fixed_tm = frame_audio(wav_data)\n \n frame = 5\n all_logits, all_embeddings = model.infer_tf(fixed_tm[:1])\n for window in fixed_tm[1:]:\n if frame_limit_secs and frame > frame_limit_secs:\n continue\n \n logits, embeddings = model.infer_tf(window[np.newaxis, :])\n all_logits = np.concatenate([all_logits, logits], axis=0)\n frame += 5\n \n frame = 5\n all_probabilities = []\n for frame_logits in all_logits:\n probabilities = tf.nn.softmax(frame_logits).numpy()\n \n ## set the appropriate row in the sample submission\n sample_submission.loc[sample_submission.row_id == file_id + \"_\" + str(frame), competition_classes] = probabilities[competition_class_map]\n frame += 5","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:57.887656Z","iopub.execute_input":"2024-05-18T13:16:57.888067Z","iopub.status.idle":"2024-05-18T13:16:57.89815Z","shell.execute_reply.started":"2024-05-18T13:16:57.888025Z","shell.execute_reply":"2024-05-18T13:16:57.897015Z"},"trusted":true},"outputs":[],"execution_count":11},{"cell_type":"markdown","source":"## Step 6: Generate a submission\n\nNow we process all of the test samples as discussed above, creating output rows, and saving them in the provided `sample_submission.csv`. Finally, we save these rows to our final output file: `submission.csv`. This is the file that gets submitted and scored when you submit the notebook.","metadata":{}},{"cell_type":"code","source":"test_samples = list(glob.glob(\"/kaggle/input/birdclef-2023/test_soundscapes/*.ogg\"))\ntest_samples","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:57.89934Z","iopub.execute_input":"2024-05-18T13:16:57.899621Z","iopub.status.idle":"2024-05-18T13:16:57.965233Z","shell.execute_reply.started":"2024-05-18T13:16:57.899593Z","shell.execute_reply":"2024-05-18T13:16:57.964304Z"},"trusted":true},"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"['/kaggle/input/birdclef-2023/test_soundscapes/soundscape_29201.ogg']"},"metadata":{}}],"execution_count":12},{"cell_type":"code","source":"sample_sub = pd.read_csv(\"/kaggle/input/birdclef-2023/sample_submission.csv\")\nsample_sub[competition_classes] = sample_sub[competition_classes].astype(np.float32)\nsample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:57.966522Z","iopub.execute_input":"2024-05-18T13:16:57.96683Z","iopub.status.idle":"2024-05-18T13:16:58.068321Z","shell.execute_reply.started":"2024-05-18T13:16:57.966805Z","shell.execute_reply":"2024-05-18T13:16:58.067382Z"},"trusted":true},"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":" row_id abethr1 abhori1 abythr1 afbfly1 afdfly1 afecuc1 \\\n0 soundscape_29201_5 0.0 0.0 0.0 0.0 0.0 0.0 \n1 soundscape_29201_10 0.0 0.0 0.0 0.0 0.0 0.0 \n2 soundscape_29201_15 0.0 0.0 0.0 0.0 0.0 0.0 \n\n affeag1 afgfly1 afghor1 ... yebsto1 yeccan1 yefcan yelbis1 yenspu1 \\\n0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 \n1 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 \n2 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0 \n\n yertin1 yesbar1 yespet1 yetgre1 yewgre1 \n0 0.0 0.0 0.0 0.0 0.0 \n1 0.0 0.0 0.0 0.0 0.0 \n2 0.0 0.0 0.0 0.0 0.0 \n\n[3 rows x 265 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
row_idabethr1abhori1abythr1afbfly1afdfly1afecuc1affeag1afgfly1afghor1...yebsto1yeccan1yefcanyelbis1yenspu1yertin1yesbar1yespet1yetgre1yewgre1
0soundscape_29201_50.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
1soundscape_29201_100.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
2soundscape_29201_150.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n

3 rows × 265 columns

\n
"},"metadata":{}}],"execution_count":13},{"cell_type":"code","source":"frame_limit_secs = 15 if sample_sub.shape[0] == 3 else None\nfor sample_filename in test_samples:\n predict_for_sample(sample_filename, sample_sub, frame_limit_secs=15)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:16:58.069415Z","iopub.execute_input":"2024-05-18T13:16:58.069694Z","iopub.status.idle":"2024-05-18T13:17:01.002686Z","shell.execute_reply.started":"2024-05-18T13:16:58.069667Z","shell.execute_reply":"2024-05-18T13:17:01.001819Z"},"trusted":true},"outputs":[],"execution_count":14},{"cell_type":"code","source":"sample_sub","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:17:01.003792Z","iopub.execute_input":"2024-05-18T13:17:01.0041Z","iopub.status.idle":"2024-05-18T13:17:01.032908Z","shell.execute_reply.started":"2024-05-18T13:17:01.004072Z","shell.execute_reply":"2024-05-18T13:17:01.031711Z"},"trusted":true},"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":" row_id abethr1 abhori1 abythr1 afbfly1 afdfly1 \\\n0 soundscape_29201_5 0.000003 0.000006 0.000065 0.000002 0.000002 \n1 soundscape_29201_10 0.000011 0.000309 0.000013 0.000326 0.000013 \n2 soundscape_29201_15 0.000219 0.000743 0.000439 0.000048 0.000005 \n\n afecuc1 affeag1 afgfly1 afghor1 ... yebsto1 yeccan1 \\\n0 0.000025 0.000001 5.608475e-07 0.000053 ... 1.832905e-07 0.000003 \n1 0.000013 0.000032 1.591020e-06 0.000003 ... 1.856587e-06 0.000068 \n2 0.000091 0.000002 2.221404e-05 0.000005 ... 2.375805e-06 0.000027 \n\n yefcan yelbis1 yenspu1 yertin1 yesbar1 yespet1 \\\n0 0.000003 1.267385e-07 1.634960e-07 9.799186e-07 0.000001 0.000001 \n1 0.000081 5.081049e-04 1.915185e-06 3.247317e-05 0.000012 0.000143 \n2 0.000321 4.050651e-05 1.578008e-05 1.949996e-05 0.000002 0.004949 \n\n yetgre1 yewgre1 \n0 5.377726e-07 0.000001 \n1 4.499636e-06 0.000263 \n2 3.138859e-05 0.000054 \n\n[3 rows x 265 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
row_idabethr1abhori1abythr1afbfly1afdfly1afecuc1affeag1afgfly1afghor1...yebsto1yeccan1yefcanyelbis1yenspu1yertin1yesbar1yespet1yetgre1yewgre1
0soundscape_29201_50.0000030.0000060.0000650.0000020.0000020.0000250.0000015.608475e-070.000053...1.832905e-070.0000030.0000031.267385e-071.634960e-079.799186e-070.0000010.0000015.377726e-070.000001
1soundscape_29201_100.0000110.0003090.0000130.0003260.0000130.0000130.0000321.591020e-060.000003...1.856587e-060.0000680.0000815.081049e-041.915185e-063.247317e-050.0000120.0001434.499636e-060.000263
2soundscape_29201_150.0002190.0007430.0004390.0000480.0000050.0000910.0000022.221404e-050.000005...2.375805e-060.0000270.0003214.050651e-051.578008e-051.949996e-050.0000020.0049493.138859e-050.000054
\n

3 rows × 265 columns

\n
"},"metadata":{}}],"execution_count":15},{"cell_type":"code","source":"sample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-18T13:17:01.034318Z","iopub.execute_input":"2024-05-18T13:17:01.034993Z","iopub.status.idle":"2024-05-18T13:17:01.054496Z","shell.execute_reply.started":"2024-05-18T13:17:01.034951Z","shell.execute_reply":"2024-05-18T13:17:01.053744Z"},"trusted":true},"outputs":[],"execution_count":16},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}