{"nbformat_minor": 1, "cells": [{"cell_type": "markdown", "source": ["Code that runs FFTs of several window sizes, aligns their centers, and then applies mel weighting to combine them. \n", "\n", "With single FFTs, short windows have good time resolution but lack frequency breadth (no lower frequencies), whereas long windows have good frequency breadth but lack time precision (windows contain many wavelengths at higher frequencies). Here we combine FFTs of varying window length to tackle this. \n", "\n", "This produces a [time x log frequency] matrix of log powers. This representation should be more invariant to distortions of both frequency and time due to the mel frequency averaging and time-window averaging respectively. \n", "\n", "Uses tensorflows eager mode (which I couldn't get to work in a kaggle kernel)"], "metadata": {"_cell_guid": "17864fab-15dd-45ac-8e4c-fe8e50911a17", "_uuid": "cf9d54e63f5e715b061fbdc3108be00a09445053"}}, {"cell_type": "markdown", "source": ["![Tensorboard](https://i.imgur.com/P5S4wHB.png)\n"], "metadata": {"_cell_guid": "5acfa777-4e86-48a4-8a85-2ad57be6c682", "_uuid": "27858bfd1eaef4c06ea2df2eae33473be0470489"}}, {"execution_count": null, "source": ["import numpy as np\n", "import librosa.display\n", "import tensorflow as tf\n", "import tensorflow.contrib.eager as tfe\n", "\n", "tfe.enable_eager_execution()\n", "\n", "from scipy.io import wavfile\n", "\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "24a44819-896b-4e53-b38d-78075f4bfe14", "collapsed": true, "_uuid": "314b1cc0c281f42fb05ccac0054c8e464b456032"}}, {"execution_count": null, "source": ["def hz_to_mel(freq):\n", "  return 1127. * tf.log(1.0 + (freq / 700.))\n", "\n", "def mel_to_hz(mel):\n", "  return 700.*(tf.exp(mel/1127.)-1.)\n", "\n", "def multi_ffts_to_mel(freq_array, n_mels=128):\n", "  melfreq_array = tf.expand_dims(hz_to_mel(freq_array),0)\n", "  \n", "  mel_edges = tf.lin_space(hz_to_mel(tf.reduce_min(freq_array)), #or just use 0\n", "                           hz_to_mel(tf.reduce_max(freq_array)), #or SR/2\n", "                           n_mels+2)\n", "  \n", "  lower_edge_mel, center_mel, upper_edge_mel =tf.split(tf.contrib.signal.frame(mel_edges, 3, 1, axis=-1), 3, axis=-1)\n", "\n", "  wt_down = (melfreq_array - lower_edge_mel) / (center_mel - lower_edge_mel)\n", "  wt_up = (upper_edge_mel - melfreq_array) / (upper_edge_mel - center_mel)\n", "  \n", "  mel_weights_matrix = tf.maximum(0.0, tf.minimum(wt_down, wt_up))\n", "  center_mel_freqs = mel_to_hz(center_mel) \n", "  \n", "  return mel_weights_matrix, center_mel_freqs\n", "\n", "def audioframes2logmelspec(b_framed_signal, n_ffts=5, \n", "                           wvls_per_window_hinge=16, n_mel=128, \n", "                           fft_l1=1024, sr=16000):\n", "  # batch_framed_signal has shape: (batch_size x n_windows x fft_l1)\n", "  # decrease weights for samples w/ more than wvls_per_window_hinge\n", "  # wvls_per_window_hinge method could be improved, maybe weight~pmf of poisson?\n", "    \n", "  fft1_space = tf.lin_space(0., .5, 1+fft_l1//2)[1:]\n", "  freq_list =[sr*fft1_space] \n", "  n_wv_list =[fft_l1*fft1_space]\n", "\n", "  fft_list =[tf.spectral.rfft(b_framed_signal)[:,:,1:]]\n", "  \n", "  for i in range(1,n_ffts):\n", "    fft_lnew = fft_l1//2**i\n", "    fftnew_space = tf.lin_space(0., .5, 1+fft_lnew//2)[1:]\n", "    \n", "    freq_list.append(sr*fftnew_space)\n", "    n_wv_list.append(fft_lnew*fftnew_space)\n", "    \n", "    frames_new = b_framed_signal[:, :, (fft_l1-fft_lnew)//2:(fft_l1-fft_lnew)//2+fft_lnew]\n", "    fft_list.append(tf.spectral.rfft(frames_new)[:,:,1:])\n", "    \n", "  \n", "  freq_concat = tf.concat(freq_list, axis=-1)\n", "  n_wv_concat = tf.concat(n_wv_list, axis=-1)\n", "  fft_concat = tf.concat(fft_list, axis=-1)\n", "    \n", "  magnitude_spectros = tf.abs(fft_concat)\n", "\n", "  mel_wts, center_mel_freqs = multi_ffts_to_mel(freq_concat, n_mel)\n", "  wvls_wts = tf.where(n_wv_concat>wvls_per_window_hinge, wvls_per_window_hinge/n_wv_concat, tf.ones_like(n_wv_concat))\n", "  \n", "  mel_spectro=tf.tensordot(magnitude_spectros, (mel_wts*tf.expand_dims(wvls_wts,0)),axes = [[2], [1]])\n", "\n", "  log_mel_spectro = tf.log(mel_spectro+1e-7)\n", "  \n", "  return tf.expand_dims(log_mel_spectro, -1), center_mel_freqs\n"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "e54a85ee-a80e-417f-a43c-97513080bdb9", "collapsed": true, "_uuid": "3cf3870e3a1abe7ee0f91627d526ddb8b5a1dc73"}}, {"execution_count": null, "source": ["some_paths = [\n", "'./data/train/audio/marvin/8625475c_nohash_0.wav',\n", "'./data/train/audio/tree/8625475c_nohash_1.wav',  \n", "'./data/train/audio/tree/8625475c_nohash_2.wav',   \n", "'./data/train/audio/tree/8625475c_nohash_3.wav',\n", "'./data/train/audio/no/8625475c_nohash_0.wav', \n", "'./data/train/audio/zero/8625475c_nohash_0.wav',\n", "'./data/train/audio/zero/8625475c_nohash_1.wav',\n", "'./data/train/audio/down/8625475c_nohash_0.wav']"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "a66bc62b-87ca-4034-9ec7-42d97e1e0592", "collapsed": true, "_uuid": "5621ddf5b86a54910f6c6f68a534ce8949f54021"}}, {"execution_count": null, "source": ["def plot_several_logmelspec(paths):\n", "  n=len(paths)\n", "\n", "  plt.figure(figsize=(12,4*n))\n", "\n", "  for i, path in enumerate(paths):\n", "    plt.subplot(n, 1, i+1)\n", "\n", "    sr, wav = wavfile.read(path)\n", "    signal = wav.astype(np.float32) / np.iinfo(np.int16).max\n", "\n", "    b_signals = tf.expand_dims(signal, axis=0)\n", "\n", "    b_framed_signal = tf.contrib.signal.frame(b_signals, \n", "                                          frame_length=1024, \n", "                                          frame_step = 32)\n", "    log_mel_spectro, center_mel_freqs = audioframes2logmelspec(b_framed_signal, sr=sr)\n", "\n", "    librosa.display.specshow(log_mel_spectro[0,:,:,0].numpy().T, sr=sr, x_axis='time', \n", "                             y_axis='mel', hop_length=32, \n", "                             fmin=tf.reduce_min(center_mel_freqs), \n", "                             fmax=tf.reduce_max(center_mel_freqs), \n", "                             cmap='coolwarm')\n", "\n", "    plt.title(path)\n", "    plt.colorbar(format='%+02.0f dB')\n", "\n", "  plt.tight_layout()"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "2fd3eb3e-15a1-4298-b7b1-7ef7d9cdbd73", "collapsed": true, "_uuid": "1419fa61cb518c7af20a09ff1d160d4a0ba55ea2"}}, {"execution_count": null, "source": ["plot_several_logmelspec(some_paths)"], "cell_type": "code", "outputs": [], "metadata": {"_cell_guid": "4450ec0e-6c2a-4319-8898-e51c4b69a5bb", "collapsed": true, "_uuid": "b384ff9e5e37b3cab685310136836935deea4e16", "scrolled": false}}, {"cell_type": "markdown", "source": ["![high_res_melspectros](https://i.imgur.com/P5S4wHB.png)"], "metadata": {"_cell_guid": "52159d2a-83b9-4ab4-95af-92110c7a3777", "_uuid": "679c79d8fc1e9fb43e1c2ffec5ce6ab293cb9cb9"}}, {"cell_type": "markdown", "source": [], "metadata": {"_cell_guid": "93a4a7da-b25b-4d40-b527-0729cbeee369", "_uuid": "6d4a805fc4a8f8b163afad210364625f941db70c"}}], "nbformat": 4, "metadata": {"language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "version": "3.6.3", "mimetype": "text/x-python", "name": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "nbconvert_exporter": "python"}, "kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}}}