{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **INF-0619 - SPEECH RECOGNITION**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Imports:","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom os.path import isdir, join\nfrom pathlib import Path\nimport pandas as pd\n\n# Math\nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nimport librosa\n\nfrom sklearn.decomposition import PCA\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport IPython.display as ipd\nimport librosa.display\n\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport pandas as pd\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pyunpack\n!pip install patool\nos.system('apt-get install p7zip')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from pyunpack import Archive\nimport shutil\nif not os.path.exists('/kaggle/working/train/'):\n    os.makedirs('/kaggle/working/train/')\nArchive('../input/tensorflow-speech-recognition-challenge/train.7z').extractall('/kaggle/working/train/')\nfor dirname, _, filenames in os.walk('/kaggle/working/train/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_audio_path = '../working/train/train/audio/'\nfilename = '/yes/0a7c2a8d_nohash_0.wav'\nsample_rate, samples = wavfile.read(str(train_audio_path) + filename)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"scipy.signal.spectrogram","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def log_specgram(audio, sample_rate, window_size=20,\n                 step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    freqs, times, spec = signal.spectrogram(audio,\n                                    fs=sample_rate,\n                                    window='hann',\n                                    nperseg=nperseg,\n                                    noverlap=noverlap,\n                                    detrend=False)\n    return freqs, times, np.log(spec.T.astype(np.float32) + eps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freqs, times, spectrogram = log_specgram(samples, sample_rate)\n\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + filename)\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)\n\nax2 = fig.add_subplot(212)\nax2.imshow(spectrogram.T, aspect='auto', origin='lower', \n           extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nax2.set_yticks(freqs[::16])\nax2.set_xticks(times[::16])\nax2.set_title('Spectrogram of ' + filename)\nax2.set_ylabel('Freqs in Hz')\nax2.set_xlabel('Seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(samples.shape)\nprint(samples[0])\nprint(sample_rate)\n\nprint(freqs.shape)\nprint(freqs[0])\nprint(times[0])\nprint(spectrogram.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![sampling](http://www.teachwithict.com/uploads/5/5/8/2/5582303/119809133_1.jpg)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(samples, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"samples_cut = samples[6000:13000]\nsample_rate_cut = 7000\n\nfreqs, times, spectrogram = log_specgram(samples_cut, sample_rate)\n\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + filename)\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples_cut), sample_rate_cut), samples_cut)\n\nax2 = fig.add_subplot(212)\nax2.imshow(spectrogram.T, aspect='auto', origin='lower', \n           extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nax2.set_yticks(freqs[::16])\nax2.set_xticks(times[::16])\nax2.set_title('Spectrogram of ' + filename)\nax2.set_ylabel('Freqs in Hz')\nax2.set_xlabel('Seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ipd.Audio(samples_cut, rate=sample_rate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dirs = [f for f in os.listdir(train_audio_path) if isdir(join(train_audio_path, f))]\ndirs.sort()\nprint('Number of labels: ' + str(len(dirs)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate\nnumber_of_recordings = []\nfor direct in dirs:\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    number_of_recordings.append(len(waves))\n\n# Plot\ndata = [go.Histogram(x=dirs, y=number_of_recordings)]\ntrace = go.Bar(\n    x=dirs,\n    y=number_of_recordings,\n    marker=dict(color = number_of_recordings, colorscale='icefire', showscale=True),\n)\nlayout = go.Layout(\n    title='Number of recordings in given label',\n    xaxis = dict(title='Words'),\n    yaxis = dict(title='Number of recordings')\n)\npy.iplot(go.Figure(data=[trace], layout=layout))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def custom_fft(y, fs):\n    T = 1.0 / fs\n    N = y.shape[0]\n    yf = fft(y)\n    xf = np.linspace(0.0, 1.0/(2.0*T), N//2)\n    vals = 2.0/N * np.abs(yf[0:N//2])  # FFT is simmetrical, so we take just the first half\n    # FFT is also complex, to we take just the real part (abs)\n    return xf, vals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames = ['on/004ae714_nohash_0.wav', 'on/0137b3f4_nohash_0.wav']\nfor filename in filenames:\n    sample_rate, samples = wavfile.read(str(train_audio_path) + filename)\n    xf, vals = custom_fft(samples, sample_rate)\n    plt.figure(figsize=(12, 4))\n    plt.title('FFT of speaker ' + filename[4:11])\n    plt.plot(xf, vals)\n    plt.xlabel('Frequency')\n    plt.grid()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Speaker ' + filenames[0][4:11])\nipd.Audio(join(train_audio_path, filenames[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Speaker ' + filenames[1][4:11])\nipd.Audio(join(train_audio_path, filenames[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filename = '/yes/01bb6a2a_nohash_1.wav'\nsample_rate, samples = wavfile.read(str(train_audio_path) + filename)\nfreqs, times, spectrogram = log_specgram(samples, sample_rate)\n\nplt.figure(figsize=(10, 7))\nplt.title('Spectrogram of ' + filename)\nplt.ylabel('Freqs')\nplt.xlabel('Time')\nplt.imshow(spectrogram.T, aspect='auto', origin='lower', \n           extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nplt.yticks(freqs[::16])\nplt.xticks(times[::16])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_of_shorter = 0\nfor direct in dirs:\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(train_audio_path + direct + '/' + wav)\n        if samples.shape[0] < sample_rate:\n            num_of_shorter += 1\nprint('Number of recordings shorter than 1 second: ' + str(num_of_shorter))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"to_keep = 'yes no up down left right on off stop go'.split()\ndirs = [d for d in dirs if d in to_keep]\n\nprint(dirs)\n\nfor direct in dirs:\n    vals_all = []\n    spec_all = []\n\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(train_audio_path + direct + '/' + wav)\n        if samples.shape[0] != 16000:\n            continue\n        xf, vals = custom_fft(samples, 16000)\n        vals_all.append(vals)\n        freqs, times, spec = log_specgram(samples, 16000)\n        spec_all.append(spec)\n\n    plt.figure(figsize=(14, 4))\n    plt.subplot(121)\n    plt.title('Mean fft of ' + direct)\n    plt.plot(np.mean(np.array(vals_all), axis=0))\n    plt.grid()\n    plt.subplot(122)\n    plt.title('Mean specgram of ' + direct)\n    plt.imshow(np.mean(np.array(spec_all), axis=0).T, aspect='auto', origin='lower', \n               extent=[times.min(), times.max(), freqs.min(), freqs.max()])\n    plt.yticks(freqs[::16])\n    plt.xticks(times[::16])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fft_all = []\nnames = []\nfor direct in dirs:\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(train_audio_path + direct + '/' + wav)\n        if samples.shape[0] != sample_rate:\n            samples = np.append(samples, np.zeros((sample_rate - samples.shape[0], )))\n        x, val = custom_fft(samples, sample_rate)\n        fft_all.append(val)\n        names.append(direct + '/' + wav)\n\nfft_all = np.array(fft_all)\n\n# Normalization\nfft_all = (fft_all - np.mean(fft_all, axis=0)) / np.std(fft_all, axis=0)\n\n# Dim reduction\npca = PCA(n_components=3)\nfft_all = pca.fit_transform(fft_all)\n\ndef interactive_3d_plot(data, names):\n    scatt = go.Scatter3d(x=data[:, 0], y=data[:, 1], z=data[:, 2], mode='markers', text=names)\n    data = go.Data([scatt])\n    layout = go.Layout(title=\"Anomaly detection\")\n    figure = go.Figure(data=data, layout=layout)\n    py.iplot(figure)\n    \ninteractive_3d_plot(fft_all, names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Recording yes/5165cf0a_nohash_0.wav')\nipd.Audio(join(train_audio_path, 'yes/5165cf0a_nohash_0.wav'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Sequential\n# define baseline model\ndef baseline_model():\n    # create model\n    model = Sequential()\n    model.add(Dense(512, input_dim=3, activation='relu'))\n    model.add(Dense(256, activation='relu'))\n    model.add(Dense(128, activation='relu'))\n    model.add(Dense(10, activation='softmax'))\n    model.summary()\n    # Compile model\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.wrappers.scikit_learn import KerasClassifier\nestimator = KerasClassifier(build_fn=baseline_model, epochs=200, batch_size=5, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\ngt = []\nfor name in names:\n    gt.append(name.split('/')[0])\n\nle = preprocessing.LabelEncoder()\nle.fit(gt)\ngt = le.transform(gt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"estimator = estimator.fit(fft_all, gt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(estimator.history)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}