{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom os.path import isdir, join\nfrom pathlib import Path\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Math\nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nimport librosa\nfrom sklearn.decomposition import PCA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport IPython.display as ipd\nimport librosa.display\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_audio_path = '/kaggle/input/tensorflow-speech-recognition-challenge/train/audio'\nfilename = 'yes/0a7c2a8d_nohash_0.wav'\nsample_rate, samples = wavfile.read(join(train_audio_path, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ll /kaggle/input/tensorflow-speech-recognition-challenge/train/audio/yes/0a7c2a8d_nohash_0.wav -h","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_rate, samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define a function that calculates spectrogram\ndef log_specgram(audio, smaple_rate, window_size=20, step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    freqs, times, spec = signal.spectrogram(audio, fs=sample_rate, window='hann', \\\n        nperseg=nperseg, noverlap=noverlap, detrend=False)\n    return freqs, times, np.log(spec.T.astype(np.float32) + eps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freqs, times, spec = log_specgram(samples, sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freqs.shape, times.shape, spec.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + filename)\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)\n\nax2 = fig.add_subplot(212)\nax2.imshow(spec.T, aspect='auto', origin='lower', \\\n    extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nax2.set_yticks(freqs[::16])\nax2.set_xticks(times[::16])\nax2.set_title('Spectrogram of ' + filename)\nax2.set_ylabel('Freqs in HZ')\nax2.set_xlabel('Seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# input normalization for NN\nmean = np.mean(spec, axis=0)\nstd = np.std(spec, axis=0)\nspectrogram = (spec - mean) / std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"spectrogram.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MFCC 梅尔频率倒谱系数\nS = librosa.feature.melspectrogram(samples.astype(float), sr=sample_rate, n_mels=128)\n# Convert to log scale (dB). we'll use peak power (max) as reference\nlog_S = librosa.power_to_db(S, ref=np.max)\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(log_S, sr=sample_rate, x_axis='time', y_axis='mel')\nplt.title('Mel power spectrogram')\nplt.colorbar(format='%+02.0f dB')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mfcc = librosa.feature.mfcc(S=log_S, n_mfcc=13)\ndelta2_mfcc = librosa.feature.delta(mfcc, order=2)\nmfcc.shape, delta2_mfcc.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nlibrosa.display.specshow(delta2_mfcc)\nplt.ylabel('MFCC coeffs')\nplt.xlabel('Time')\nplt.title('MFCC')\nplt.colorbar()\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Spectrogram in 3d\ndata = [go.Surface(x=times, y=freqs, z=spectrogram.T)]\nlayout = go.Layout(\n    title='Spectrogram of \"yes\" in 3d',\n    scene = dict(\n        yaxis = dict(title='Frequencies', range=[freqs.min(), freqs.max()]),\n        xaxis = dict(title='Time', range=[times.min(), times.max()]),\n        zaxis = dict(title='Log amplitude')\n    )\n)\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Silence removal\nprint(samples.shape)\nipd.Audio(samples, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_cut = samples[4000:13000]\nprint(sample_cut.shape)\nipd.Audio(sample_cut, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# guessed slignment of each letter in 'yes'\nfreqs, times, spectrogram_cut = log_specgram(sample_cut, sample_rate)\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + filename)\nax1.set_ylabel('Amplitude')\nax1.plot(sample_cut)\n\nax2 = fig.add_subplot(212)\nax2.set_title('Spectrogram of ' + filename)\nax2.set_ylabel('Frequencies * 0.1')\nax2.set_xlabel('Samples')\nax2.imshow(spectrogram_cut.T, aspect='auto', origin='lower',\n          extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nax2.set_yticks(freqs[::16])\nax2.set_xticks(times[::16])\nax2.text(0.06, 1000, 'Y', fontsize=18)\nax2.text(0.17, 1000, 'E', fontsize=18)\nax2.text(0.36, 1000, 'S', fontsize=18)\nxcoords = [0.025, 0.11, 0.23, 0.49]\nfor xc in xcoords:\n    ax1.axvline(x=xc*16000, c='r')\n    ax2.axvline(x=xc, c='r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Resampling - dimension reduction\nfilename = 'happy/0b09edd3_nohash_0.wav'\nnew_sample_rate = 8000\nsample_rate, samples = wavfile.read(join(train_audio_path, filename))\nresampled = signal.resample(samples, int(new_sample_rate/sample_rate*samples.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"int(new_sample_rate/sample_rate*samples.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(samples, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(resampled, rate=new_sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fast fourier transform\ndef custom_fft(y, fs):\n    T = 1.0 / fs\n    N = y.shape[0]\n    yf = fft(y)\n    xf = np.linspace(0.0, 1.0/(2.0*T), N//2)\n    vals = 2.0 / N * np.abs(yf[0:N//2])\n    return xf, vals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xf, vals = custom_fft(samples, sample_rate)\nplt.figure(figsize=(12, 4))\nplt.title('FFT of recording sampled with ' + str(sample_rate) + ' Hz')\nplt.plot(xf, vals)\nplt.xlabel('Frequency')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xf, vals = custom_fft(resampled, new_sample_rate)\nplt.figure(figsize=(12, 4))\nplt.title('FFT of recording sampled with ' + str(new_sample_rate) + ' Hz')\nplt.plot(xf, vals)\nplt.xlabel('Frequency')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Basic data statistics"},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of records\ndirs = [f for f in os.listdir(train_audio_path) if isdir(join(train_audio_path, f))]\ndirs.sort()\nprint('Number of labels: ' + str(len(dirs)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculate\nnumber_of_recordings = []\nfor direct in dirs:\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    number_of_recordings.append(len(waves))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of total recordings in train sets\nsum(number_of_recordings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot\ndata = [go.Histogram(x=dirs, y=number_of_recordings)]\ntrace = go.Bar(\n    x=dirs,\n    y=number_of_recordings,\n    marker=dict(color=number_of_recordings, colorscale='viridis', showscale=True)\n)\nlayout=go.Layout(\n    title='Number of recordings in given label',\n    xaxis=dict(title='Words'),\n    yaxis=dict(title='Number of recordings')\n)\npy.iplot(go.Figure(data=[trace], layout=layout))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Speaker doesn't occur in both train and test data sets\nfilenames = ['on/004ae714_nohash_0.wav', 'on/0137b3f4_nohash_0.wav']\nfor filename in filenames:\n    sample_rate, samples = wavfile.read(join(train_audio_path, filename))\n    xf, vals = custom_fft(samples, sample_rate)\n    plt.figure(figsize=(12, 4))\n    plt.title('FFT of speaker ' + filename[4:11])\n    plt.plot(xf, vals)\n    plt.xlabel('Frequency')\n    plt.grid()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Speaker ' + filenames[0][4:11])\nipd.Audio(join(train_audio_path, filenames[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Speaker ' + filenames[1][4:11])\nipd.Audio(join(train_audio_path, filenames[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Recordings with some weird silence\nfilename = 'yes/01bb6a2a_nohash_1.wav'\nsample_rate, samples = wavfile.read(join(train_audio_path, filename))\nfreqs, times, spectrogram = log_specgram(samples, sample_rate)\nplt.figure(figsize=(10, 7))\nplt.title('Spectrogram of ' + filename)\nplt.ylabel('Freqs')\nplt.xlabel('Times')\nplt.imshow(spectrogram.T, aspect='auto', origin='lower', \\\n    extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nplt.yticks(freqs[::16])\nplt.xticks(times[::16])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(join(train_audio_path, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculating number of recordings shorter than 1 second\nnum_of_shorter = 0\nfor d in dirs:\n    waves = [f for f in os.listdir(join(train_audio_path, d)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(join(train_audio_path, d, wav))\n        if samples.shape[0] < sample_rate:\n            num_of_shorter += 1\nprint('Number of recordings shorter than 1 second: ' + str(num_of_shorter))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mean spectrograms and FFT for each word\nto_keep = 'yes no up down left right on off stop go'.split()\ndirs = [d for d in dirs if d in to_keep]\nprint(dirs)\nfor d in dirs:\n    vals_all = []\n    spec_all = []\n    waves = [f for f in os.listdir(join(train_audio_path, d)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(join(train_audio_path, d, wav))\n        if samples.shape[0] != 16000: continue\n        xf, vals = custom_fft(samples, 16000)\n        vals_all.append(vals)\n        freqs, times, spec = log_specgram(samples, 16000)\n        spec_all.append(spec)\n    plt.figure(figsize=(14, 4))\n    plt.subplot(121)\n    plt.title('Mean fft of ' + d)\n    plt.plot(np.mean(np.array(vals_all), axis=0))\n    plt.grid()\n    plt.subplot(122)\n    plt.title('Mean spectrogram of ' + d)\n    plt.imshow(np.mean(np.array(spec_all), axis=0).T, aspect='auto', origin='lower', \\\n        extent=[times.min(), times.max(), freqs.min(), freqs.max()])\n    plt.yticks(freqs[::16])\n    plt.xticks(times[::16])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Frequenciy components across the words\ndef violinplot_frequency(dirs, freq_ind):\n    spec_all = []\n    for idx, d in enumerate(dirs):\n        spec_all.append([])\n        waves = [f for f in os.listdir(join(train_audio_path, d)) if f.endswith('.wav')]\n        for wav in waves[:100]:\n            sample_rate, samples = wavfile.read(join(train_audio_path, d, wav))\n            freqs, times, spec = log_specgram(samples, sample_rate)\n            spec_all[idx].extend(spec[:, freq_ind])\n    minimum = min([len(spec) for spec in spec_all])\n    spec_all = np.array([spec[:minimum] for spec in spec_all])\n    plt.figure(figsize=(13, 7))\n    plt.title('Frequency ' + str(freqs[freq_ind]) + ' Hz')\n    plt.ylabel('Amount of frequency in a word')\n    plt.xlabel('Words')\n    sns.violinplot(data=pd.DataFrame(spec_all.T, columns=dirs))\n    plt.show()      ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"violinplot_frequency(dirs, 20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"violinplot_frequency(dirs, 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"violinplot_frequency(dirs, 120)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Anomaly detection\nfft_all = []\nnames = []\nfor d in dirs:\n    waves = [f for f in os.listdir(join(train_audio_path, d)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(join(train_audio_path, d, wav))\n        if samples.shape[0] != sample_rate:\n            samples = np.append(samples, np.zeros((sample_rate - samples.shape[0],)))\n        x, val = custom_fft(samples, sample_rate)\n        fft_all.append(val)\n        names.append(join(d, wav))\nfft_all = np.array(fft_all)\n\n# Normalization\nfft_all = (fft_all - np.mean(fft_all, axis=0)) / np.std(fft_all, axis=0)\n\n# Dimension reduction\npca = PCA(n_components=3)\nfft_all = pca.fit_transform(fft_all)\n\ndef interactive_3d_plot(data, names):\n    scatt = go.Scatter3d(x=data[:, 0], y=data[:, 1], z=data[:, 2], mode='markers', text=names)\n    data = go.Data([scatt])\n    layout = go.Layout(title='Anomaly detection')\n    figure = go.Figure(data=data, layout=layout)\n    py.iplot(figure)\ninteractive_3d_plot(fft_all, names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(fft_all)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Anomaly samples\nprint('Recording go/0487ba9b_nohash_0.wav')\nipd.Audio(join(train_audio_path, 'go/0487ba9b_nohash_0.wav'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Recording yes/e4b02540_nohash_0.wav')\nipd.Audio(join(train_audio_path, 'yes/e4b02540_nohash_0.wav'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Recording seven/b1114e4f_nohash_0.wav')\nipd.Audio(join(train_audio_path, 'seven/b1114e4f_nohash_0.wav'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. possible solutions based on above analysis"},{"metadata":{},"cell_type":"markdown","source":"1. Encoder-decoder\n2. RNNs with CTC loss\n3. Classic speech recognition approach like Kaldi.\n4. Deep CNN ✔"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}