{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\nIn this notebook, I show you how to generate time-series signals and image spectrogram data from audio. This notebook contains several functions making it easy for you to use to generate a dataset. \n\nYou can use the amplitude signal or fft in an RNN, mel spectrogram with a CNN (the most popular and usually successful approach), or try some uncommon feature engineering with perhaps the tempogram to see if it works. \n\nIn this notebook, I show you how to generate and visualize several features, and hopefully, this will help you understand audio data and preprocessing better.\n\n### Features\nThe following features will be explored in this notebook\n- amplitude signal\n- power spectrum - fft\n- spectrogram - stft\n- mel spectrogram \n- tempo\n- tempogram\n- mfcc with different deltas\n\n### Sources\nSome code was inspired by or borrowed from the following sources\n- https://github.com/musikalkemist/DeepLearningForAudioWithPython\n- https://www.kaggle.com/kaerunantoka/birdclef2022-create-image-data-from-audio-data/notebook\n- https://librosa.org/doc/latest/index.html","metadata":{}},{"cell_type":"markdown","source":"# Imports and Config","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport scipy.stats\nimport json\nimport glob\nimport soundfile as sf\nimport librosa\nimport librosa.display\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom soundfile import SoundFile\nimport IPython.display as ipd\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nplt.style.use('ggplot')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-18T20:46:00.31411Z","iopub.execute_input":"2022-02-18T20:46:00.3149Z","iopub.status.idle":"2022-02-18T20:46:00.324926Z","shell.execute_reply.started":"2022-02-18T20:46:00.314833Z","shell.execute_reply":"2022-02-18T20:46:00.323671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    SR = 32_000\n    duration = 5\n    \n    n_fft = 2048\n    hop_length = n_fft // 4\n    \n    n_mels = 224\n    fmin = 20\n    fmax = 16000","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:00.866829Z","iopub.execute_input":"2022-02-18T20:46:00.867641Z","iopub.status.idle":"2022-02-18T20:46:00.872254Z","shell.execute_reply.started":"2022-02-18T20:46:00.867597Z","shell.execute_reply":"2022-02-18T20:46:00.871557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/birdclef-2022/train_metadata.csv\")\ntrain_paths = glob.glob(\"../input/birdclef-2022/train_audio/*\")\nprint(f\"example path: {train_paths[0]}\\n\")\nprint(f\"number of birds: {len(train_paths)}\\n\")\nprint(f\"metadata columns: {train.columns.values}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:01.585909Z","iopub.execute_input":"2022-02-18T20:46:01.586829Z","iopub.status.idle":"2022-02-18T20:46:01.672644Z","shell.execute_reply.started":"2022-02-18T20:46:01.586787Z","shell.execute_reply":"2022-02-18T20:46:01.671976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:01.942594Z","iopub.execute_input":"2022-02-18T20:46:01.943264Z","iopub.status.idle":"2022-02-18T20:46:01.965133Z","shell.execute_reply.started":"2022-02-18T20:46:01.943219Z","shell.execute_reply":"2022-02-18T20:46:01.963982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This is just a random file I chose, you can look at any\nEX_FILE = \"../input/birdclef-2022/train_audio/cacgoo1/XC144036.ogg\"","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:02.180717Z","iopub.execute_input":"2022-02-18T20:46:02.18138Z","iopub.status.idle":"2022-02-18T20:46:02.185837Z","shell.execute_reply.started":"2022-02-18T20:46:02.181327Z","shell.execute_reply":"2022-02-18T20:46:02.185004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Listen","metadata":{}},{"cell_type":"code","source":"ipd.Audio(EX_FILE)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:02.689595Z","iopub.execute_input":"2022-02-18T20:46:02.690287Z","iopub.status.idle":"2022-02-18T20:46:02.729538Z","shell.execute_reply.started":"2022-02-18T20:46:02.690237Z","shell.execute_reply":"2022-02-18T20:46:02.728834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Audio Signal - Time-Amplitude\n\nAmplitude Signal - get the audio signal as a list of floating point values representing the amplitude of the signal at any given time (Amplitude being the sound wave measured from its equilibrium position).","metadata":{}},{"cell_type":"code","source":"def get_signal(path):\n    \"\"\" Get audio signal from librosa \"\"\"\n    signal, sr = librosa.load(path, sr=CFG.SR)\n    return signal\n\ndef plot_signal(signal):\n    \"\"\" Plots the time-amplitude graph of the audio signal \"\"\"\n    plt.figure(figsize=(12,8))\n    librosa.display.waveshow(signal, sr=CFG.SR)\n    plt.xlabel(\"Time\")\n    plt.ylabel(\"Amplitude\")\n    plt.title(\"Audio Signal - Time-Amplitude\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:03.309249Z","iopub.execute_input":"2022-02-18T20:46:03.309648Z","iopub.status.idle":"2022-02-18T20:46:03.315054Z","shell.execute_reply.started":"2022-02-18T20:46:03.309617Z","shell.execute_reply":"2022-02-18T20:46:03.314141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal = get_signal(EX_FILE)\nplot_signal(signal)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:03.609474Z","iopub.execute_input":"2022-02-18T20:46:03.609992Z","iopub.status.idle":"2022-02-18T20:46:04.208386Z","shell.execute_reply.started":"2022-02-18T20:46:03.609946Z","shell.execute_reply":"2022-02-18T20:46:04.202764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Power Spectrum - FFT\nFast-Fourier Transform - used to create a power spectrum by computing the magnitude and frequency of a signal","metadata":{}},{"cell_type":"code","source":"def calculate_mag_freq(signal):\n    \"\"\" Computes the magnitude and frequency given a signal \"\"\"\n    fft = np.fft.fft(signal)\n\n    # calculate abs values on complex numbers to get magnitude\n    spectrum = np.abs(fft)\n\n    # create frequency variable\n    f = np.linspace(0, CFG.SR, len(spectrum))\n\n    # only half the plot is needed because the graph is symetrical\n    left_spectrum = spectrum[:int(len(spectrum)/2)]\n    left_freq = f[:int(len(spectrum)/2)]\n    \n    return left_spectrum, left_freq\n\ndef plot_spectrum(magnitude, freq):\n    \"\"\" Plots a spectrum given a magnitude and frequency \"\"\"\n    plt.figure(figsize=(12, 8))\n    plt.plot(freq, magnitude)\n    plt.xlabel(\"Frequency\")\n    plt.ylabel(\"Magnitude\")\n    plt.title(\"Power Spectrum\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:04.331976Z","iopub.execute_input":"2022-02-18T20:46:04.332264Z","iopub.status.idle":"2022-02-18T20:46:04.339616Z","shell.execute_reply.started":"2022-02-18T20:46:04.332231Z","shell.execute_reply":"2022-02-18T20:46:04.338796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"magnitude, frequency = calculate_mag_freq(signal)\nplot_spectrum(magnitude, frequency)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:04.691413Z","iopub.execute_input":"2022-02-18T20:46:04.692048Z","iopub.status.idle":"2022-02-18T20:46:10.115987Z","shell.execute_reply.started":"2022-02-18T20:46:04.692003Z","shell.execute_reply":"2022-02-18T20:46:10.115088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectrogram - STFT\nShort-Time Fourier Transform - used to create a spectrogram, then I convert the spectrogram from amplitude to decibels to calculate the log spectrogram","metadata":{}},{"cell_type":"code","source":"def calculate_spectrogram(signal):\n    \"\"\" Computes a log spectrogram given a signal \"\"\"\n    stft = librosa.stft(signal, n_fft=CFG.n_fft, hop_length=CFG.hop_length)\n    spectrogram = np.abs(stft)\n    log_spectrogram = librosa.amplitude_to_db(spectrogram)\n    return log_spectrogram\n\ndef plot_spectrogram(log_spectrogram):\n    \"\"\" Plots a spectrogram \"\"\"\n    plt.figure(figsize=(12, 8))\n    librosa.display.specshow(log_spectrogram, sr=CFG.SR, hop_length=CFG.hop_length)\n    plt.xlabel(\"Time\")\n    plt.ylabel(\"Frequency\")\n    plt.colorbar(format=\"%+2.0f dB\")\n    plt.title(\"Spectrogram (dB)\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:10.118085Z","iopub.execute_input":"2022-02-18T20:46:10.118314Z","iopub.status.idle":"2022-02-18T20:46:10.124824Z","shell.execute_reply.started":"2022-02-18T20:46:10.118286Z","shell.execute_reply":"2022-02-18T20:46:10.123786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_spec = calculate_spectrogram(signal)\nplot_spectrogram(log_spec)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:10.126448Z","iopub.execute_input":"2022-02-18T20:46:10.127308Z","iopub.status.idle":"2022-02-18T20:46:17.293645Z","shell.execute_reply.started":"2022-02-18T20:46:10.127259Z","shell.execute_reply":"2022-02-18T20:46:17.292878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mel Spectrogram\nMel Spectrogram - very popular and effective image data that provides our models with sound information similar to what a human would perceive","metadata":{}},{"cell_type":"code","source":"def calculate_melspec(signal):\n    \"\"\" Computes a mel spectrogram \"\"\"\n    melspec = librosa.feature.melspectrogram(\n        y=signal, sr=CFG.SR, n_mels=CFG.n_mels, fmin=CFG.fmin, fmax=CFG.fmax,\n    )\n\n    melspec = librosa.power_to_db(melspec).astype(np.float32)\n    return melspec\n\ndef plot_melspec(melspec):\n    \"\"\" Plots a mel spectrogram \"\"\"\n    plt.figure(figsize=(12,8))\n    img = librosa.display.specshow(melspec, x_axis=\"time\",\n                                   y_axis=\"mel\", sr=CFG.SR,\n                                   fmax=CFG.fmax) \n    plt.colorbar(img, format=\"%+2.0f dB\")\n    plt.title(\"Mel-Frequency Spectrogram\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:17.295638Z","iopub.execute_input":"2022-02-18T20:46:17.296151Z","iopub.status.idle":"2022-02-18T20:46:17.305768Z","shell.execute_reply.started":"2022-02-18T20:46:17.296111Z","shell.execute_reply":"2022-02-18T20:46:17.304132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"melspec = calculate_melspec(signal)\nplot_melspec(melspec)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:17.307779Z","iopub.execute_input":"2022-02-18T20:46:17.30826Z","iopub.status.idle":"2022-02-18T20:46:19.619864Z","shell.execute_reply.started":"2022-02-18T20:46:17.308212Z","shell.execute_reply":"2022-02-18T20:46:19.618895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Additional Features\nTempo - the speed which the sound is played as a tempo\n\nTempogram - a image representation of tempo information\n\nmfcc delta - Mel-frequency cepstral coefficients at different deltas computed based on Savitsky-Golay filtering.","metadata":{}},{"cell_type":"code","source":"def calculate_tempo(signal):\n    \"\"\" Computes the tempo of an audio signal \"\"\"\n    onset_env = librosa.onset.onset_strength(y=signal, sr=CFG.SR)\n    tempo = librosa.beat.tempo(onset_envelope=onset_env, sr=CFG.SR)\n    return tempo\n\ndef plot_tempogram(signal):\n    \"\"\" PLots a autocorrelation tempogram \"\"\"\n    oenv = librosa.onset.onset_strength(y=signal, sr=CFG.SR, hop_length=CFG.hop_length)\n    tempogram = librosa.feature.fourier_tempogram(onset_envelope=oenv, sr=CFG.SR,\n                                                  hop_length=CFG.hop_length)\n    # Compute the auto-correlation tempogram, unnormalized to make comparison easier\n    ac_tempogram = librosa.feature.tempogram(onset_envelope=oenv, sr=CFG.SR,\n                                             hop_length=CFG.hop_length, norm=None)\n                             \n    plt.figure(figsize=(8, 4))\n    librosa.display.specshow(ac_tempogram, sr=CFG.SR, hop_length=CFG.hop_length,\n                         x_axis=\"time\", y_axis=\"tempo\", cmap=\"magma\")\n    plt.title(\"Autocorrelation Tempogram\")\n    plt.show()\n    \ndef mfcc_delta(signal):\n    \"\"\" Plots the mfcc, mfcc delta, and mfcc delta2 \"\"\"\n    mfcc = librosa.feature.mfcc(y=signal, sr=CFG.SR)\n    mfcc_delta = librosa.feature.delta(mfcc)\n    mfcc_delta2 = librosa.feature.delta(mfcc, order=2)\n    \n    fig, ax = plt.subplots(nrows=3, figsize=(12, 8), sharex=True, sharey=True)\n    img1 = librosa.display.specshow(mfcc, ax=ax[0], x_axis='time')\n    ax[0].set(title='MFCC')\n    ax[0].label_outer()\n    img2 = librosa.display.specshow(mfcc_delta, ax=ax[1], x_axis='time')\n    ax[1].set(title=r'MFCC-$\\Delta$')\n    ax[1].label_outer()\n    img3 = librosa.display.specshow(mfcc_delta2, ax=ax[2], x_axis='time')\n    ax[2].set(title=r'MFCC-$\\Delta^2$')\n    fig.colorbar(img1, ax=[ax[0]])\n    fig.colorbar(img2, ax=[ax[1]])\n    fig.colorbar(img3, ax=[ax[2]])\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:19.621298Z","iopub.execute_input":"2022-02-18T20:46:19.621582Z","iopub.status.idle":"2022-02-18T20:46:19.639354Z","shell.execute_reply.started":"2022-02-18T20:46:19.621542Z","shell.execute_reply":"2022-02-18T20:46:19.638768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"tempo:\", calculate_tempo(signal))","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:19.640292Z","iopub.execute_input":"2022-02-18T20:46:19.640922Z","iopub.status.idle":"2022-02-18T20:46:20.918072Z","shell.execute_reply.started":"2022-02-18T20:46:19.640891Z","shell.execute_reply":"2022-02-18T20:46:20.917105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_tempogram(signal)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:20.919436Z","iopub.execute_input":"2022-02-18T20:46:20.920055Z","iopub.status.idle":"2022-02-18T20:46:25.06683Z","shell.execute_reply.started":"2022-02-18T20:46:20.920008Z","shell.execute_reply":"2022-02-18T20:46:25.06592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mfcc_delta(signal)","metadata":{"execution":{"iopub.status.busy":"2022-02-18T20:46:25.06814Z","iopub.execute_input":"2022-02-18T20:46:25.068416Z","iopub.status.idle":"2022-02-18T20:46:26.81997Z","shell.execute_reply.started":"2022-02-18T20:46:25.068379Z","shell.execute_reply":"2022-02-18T20:46:26.818771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}