{"nbformat_minor": 1, "cells": [{"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "51a3fbf3a55f8db57140c6e7d2cd567bef1119bf", "_cell_guid": "2db39f1f-7b53-41d7-9f72-3114eb54f0a9"}, "source": ["# importing dependencies\n", "import pandas as pd # data frame\n", "import numpy as np # matrix math\n", "from scipy.io import wavfile # reading the wavfile\n", "import os # interation with the OS\n", "from sklearn.utils import shuffle # shuffling of data\n", "from random import sample # random selection\n", "from tqdm import tqdm # progress bar\n", "import matplotlib.pyplot as plt # to view graphs\n", "\n", "# audio processing\n", "from scipy import signal # audio processing\n", "from scipy.fftpack import dct\n", "import librosa # library for audio processing\n"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "032e1cb39ade61251edb4b46f0da0b92c01d021b", "_cell_guid": "60e56933-076a-417f-9aed-49e144dea361"}, "source": ["PATH = '../input/train/audio/'"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "46d8371b4dfa62f2522fdd96f243b6eb26ac4e99", "_cell_guid": "f21bbe87-a0a0-4029-9c74-df0303ef2c54"}, "source": ["def load_files(path):\n", "\t# write the complete file loading function here, this will return\n", "\t# a dataframe having files and labels\n", "\t# loading the files\n", "\ttrain_labels = os.listdir(PATH)\n", "\ttrain_labels.remove('_background_noise_')\n", "\n", "\tlabels_to_keep = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'silence']\n", "\n", "\ttrain_file_labels = dict()\n", "\tfor label in train_labels:\n", "\t\tfiles = os.listdir(PATH + '/' + label)\n", "\t\tfor f in files:\n", "\t\t\ttrain_file_labels[label + '/' + f] = label\n", "\n", "\ttrain = pd.DataFrame.from_dict(train_file_labels, orient='index')\n", "\ttrain = train.reset_index(drop=False)\n", "\ttrain = train.rename(columns={'index': 'file', 0: 'folder'})\n", "\ttrain = train[['folder', 'file']]\n", "\ttrain = train.sort_values('file')\n", "\ttrain = train.reset_index(drop=True)\n", "\n", "\tdef remove_label_from_file(label, fname):\n", "\t\treturn path + label + '/' + fname[len(label)+1:]\n", "\n", "\ttrain['file'] = train.apply(lambda x: remove_label_from_file(*x), axis=1)\n", "\ttrain['label'] = train['folder'].apply(lambda x: x if x in labels_to_keep else 'unknown')\n", "\n", "\tlabels_to_keep.append('unknown')\n", "\n", "\treturn train, labels_to_keep"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "2e22fe98869bc8c7f064ff5aea9c3ec0a1fb602c", "_cell_guid": "29d7e327-3a60-4df4-a6f5-6a652605fc36"}, "source": ["train, labels_to_keep = load_files(PATH)\n", "\n", "# making word2id dictr\n", "word2id = dict((c,i+1) for i,c in enumerate(sorted(labels_to_keep)))\n", "\n", "print(word2id)\n", "\n", "# get some files which will be labeled as unknown\n", "unk_files = train.loc[train['label'] == 'unknown']['file'].values\n", "# randomly selecting 3000 files\n", "unk_files = sample(list(unk_files), 3000)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "3a9c89a589161ac00a9195ec1d7300d9289185b8", "_cell_guid": "e229adff-ab0e-499c-bc13-201b65c9e499"}, "source": ["def log_specgram(audio, sample_rate, window_size=10, \n", "                 step_size=10, eps=1e-10):\n", "    nperseg = int(round(window_size * sample_rate / 1e3))\n", "    noverlap = int(round(step_size * sample_rate / 1e3))\n", "    _, _, spec = signal.spectrogram(audio, fs=sample_rate,\n", "                                    window='hann', nperseg=nperseg, noverlap=noverlap,\n", "                                    detrend=False)\n", "    return np.log(spec.T.astype(np.float32) + eps)\n", "\n", "def audio_to_data(path):\n", "    # we take a single path and convert it into data\n", "    sample_rate, audio = wavfile.read(path)\n", "    spectrogram = log_specgram(audio, sample_rate, 10, 0)\n", "    return spectrogram.T\n", "\n", "def paths_to_data(paths, word2id, unk = False):\n", "    data = np.zeros(shape = (len(paths), 81, 100))\n", "    labels = []\n", "    indexes = []\n", "    for i in tqdm(range(len(paths))):\n", "        f = paths[i]\n", "        audio = audio_to_data(paths[i])\n", "        if audio.shape != (81,100):\n", "            indexes.append(i)\n", "        else:\n", "            data[i] = audio\n", "        # print('Number of instances with inconsistent shape:', len(indexes))\n", "        # mode, if unk is set we are doing it for unknown files\n", "        if unk == True:\n", "            labels.append(word2id['unknown'])\n", "        else:\n", "            labels.append(word2id[f.split('/')[-2]])\n", "\n", "    return data, labels, indexes"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "0084c2f0bcfe40b13f149e1f50a6d7d5bfd2595d", "_cell_guid": "2870d788-e1b4-43bf-8df3-f6b01713f62b"}, "source": ["files = train.loc[train['label'] != 'unknown']['file'].values\n", "print(\"[!]For labled data...\")\n", "data, l, i = paths_to_data(files[:1], word2id)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "53542b130dbaa9879d37729d774472cf292d72ab", "_cell_guid": "e6aa023b-325e-4a8c-8bbf-68c5180a5515"}, "source": ["plt.figure(figsize = (10, 10))\n", "plt.imshow(data[0])\n", "plt.title('Log Spectrogram')"], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "86417e84f45bf4072431af11bbd078c8758631f0", "_cell_guid": "1132183c-4526-4ad1-8248-670255915038"}, "source": ["## MFCC and Filter Bank Sample\n", "\n", "I learned it from Haytham Fayek's excellent blog post on the topic. You can read it at http://haythamfayek.com/2016/04/21/speech-processing-for-machine-learning.html."]}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "0bcba47da642e4339ac4bba41af7ea51c8afeae6", "_cell_guid": "b3643ea4-c039-4896-ab47-c5512a02f276"}, "source": ["def mfcc_features(path_file, frame_size, frame_stride):\n", "    sample_rate, signal = wavfile.read(path_file)\n", "    pre_emphasis = 0.97\n", "    emphasized_signal = np.append(signal[0], signal[1:] - pre_emphasis * signal[:-1])\n", "\n", "    # params\n", "    '''frame_size = 0.025\n", "    frame_stride = 0.01'''\n", "    frame_length, frame_step = frame_size * sample_rate, frame_stride * sample_rate  # Convert from seconds to samples\n", "    signal_length = len(emphasized_signal)\n", "    frame_length = int(round(frame_length))\n", "    frame_step = int(round(frame_step))\n", "    num_frames = int(np.ceil(float(np.abs(signal_length - frame_length)) / frame_step))  # Make sure that we have at least 1 frame\n", "\n", "    pad_signal_length = num_frames * frame_step + frame_length\n", "    z = np.zeros((pad_signal_length - signal_length))\n", "    pad_signal = np.append(emphasized_signal, z) # Pad Signal to make sure that all frames have equal number of samples without truncating any samples from the original signal\n", "\n", "    indices = np.tile(np.arange(0, frame_length), (num_frames, 1)) +\\\n", "        np.tile(np.arange(0, num_frames * frame_step, frame_step), (frame_length, 1)).T\n", "    frames = pad_signal[indices.astype(np.int32, copy=False)]\n", "\n", "    # hamming window\n", "    frames *= np.hamming(frame_length)\n", "\n", "    NFFT = 512\n", "    mag_frames = np.absolute(np.fft.rfft(frames, NFFT))  # Magnitude of the FFT\n", "    pow_frames = ((1.0 / NFFT) * ((mag_frames) ** 2))  # Power Spectrum\n", "\n", "    nfilt = 40\n", "    low_freq_mel = 0\n", "    high_freq_mel = (2595 * np.log10(1 + (sample_rate / 2) / 700))  # Convert Hz to Mel\n", "    mel_points = np.linspace(low_freq_mel, high_freq_mel, nfilt + 2)  # Equally spaced in Mel scale\n", "    hz_points = (700 * (10**(mel_points / 2595) - 1))  # Convert Mel to Hz\n", "    bin = np.floor((NFFT + 1) * hz_points / sample_rate)\n", "\n", "    fbank = np.zeros((nfilt, int(np.floor(NFFT / 2 + 1))))\n", "    for m in range(1, nfilt + 1):\n", "        f_m_minus = int(bin[m - 1])   # left\n", "        f_m = int(bin[m])             # center\n", "        f_m_plus = int(bin[m + 1])    # right\n", "\n", "        for k in range(f_m_minus, f_m):\n", "            fbank[m - 1, k] = (k - bin[m - 1]) / (bin[m] - bin[m - 1])\n", "        for k in range(f_m, f_m_plus):\n", "            fbank[m - 1, k] = (bin[m + 1] - k) / (bin[m + 1] - bin[m])\n", "    filter_banks = np.dot(pow_frames, fbank.T)\n", "    filter_banks = np.where(filter_banks == 0, np.finfo(float).eps, filter_banks)  # Numerical Stability\n", "    filter_banks = 20 * np.log10(filter_banks)  # dB\n", "    \n", "    num_ceps = 20\n", "    mfcc = dct(filter_banks, type=2, axis=1, norm='ortho')[:, 1 : (num_ceps + 1)] # Keep 2-13\n", "    \n", "    cep_lifter = 22\n", "    (nframes, ncoeff) = mfcc.shape\n", "    n = np.arange(ncoeff)\n", "    lift = 1 + (cep_lifter / 2) * np.sin(np.pi * n / cep_lifter)\n", "    mfcc *= lift  #*\n", "    \n", "    return filter_banks, mfcc\n", "\n", "def normalized_fb(fb):\n", "    fb -= (np.mean(fb, axis=0) + 1e-8)\n", "    return fb\n", "\n", "def normalized_mfcc(mfcc):\n", "    mfcc -= (np.mean(mfcc, axis=0) + 1e-8)\n", "    return mfcc"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {}, "source": ["path_file = files[123]\n", "frame_size = 0.05\n", "frame_stride = 0.03\n", "fb, mfcc = mfcc_features(path_file, frame_size, frame_stride)\n", "print(fb.shape)\n", "print(mfcc.shape)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "0c610b65766eb833863c8bbd9b42544cf8d65986", "_cell_guid": "1ae8be5f-8067-40c4-9844-76b556440e63", "scrolled": false}, "source": ["# plt.figure(figsize = fb.shape)\n", "plt.imshow(fb)\n", "plt.title('Filter Bank')"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "ff7f4f85f38ca2a5b20eb2a5646bb15dc6d5607f", "_cell_guid": "ab89bc29-e059-41e4-9b60-79b94ad27034"}, "source": ["fb_n = normalized_fb(fb)\n", "# plt.figure(figsize = (25, 10))\n", "plt.imshow(fb_n)\n", "plt.title('Normalized Filter Bank')"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "c6f6a9dc5fd7ca62ff10e0c04ebecd7f1d082a1c", "_cell_guid": "50b002df-d040-4cd6-80b4-8fb9b4c07b5b"}, "source": ["# plt.figure(figsize = mfcc.shape)\n", "plt.imshow(mfcc)\n", "plt.title('MFCC')"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"_uuid": "c567d37a4700257155ef72a80828511a54aff6e4", "_cell_guid": "9cf08c9d-5807-4ce6-9532-71a68a9570f3", "scrolled": false}, "source": ["mfcc_n = normalized_mfcc(mfcc)\n", "# plt.figure(figsize = (25, 10))\n", "plt.imshow(mfcc_n)\n", "plt.title('Normalized MFCC')"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "source": ["# sometimes we "], "execution_count": null, "outputs": []}, {"cell_type": "markdown", "metadata": {"_uuid": "c99ef828238a04154add6cda5f10b1aa53927f08", "_cell_guid": "4ab8f125-b2ae-4db5-a105-00d926946697"}, "source": ["## Feature extraction using librosa"]}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "e0035c2c9940a1a76e91a231621a9e4d2fbfdcc7", "_cell_guid": "bc8ae04e-a2f4-40fe-ab2a-70368d58be38"}, "source": ["path_file = files[104]\n", "audio, sr = librosa.load(path_file)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "255585dd461aacb04cc3c55ada9306deb936c9a0", "_cell_guid": "2661edae-3844-4c93-87a0-ba463a17f053"}, "source": ["stft = np.abs(librosa.stft(audio))\n", "# mfcc\n", "mfcc_l = librosa.feature.mfcc(y=audio, sr=sr, n_mfcc=40)\n", "mfccs = np.mean(librosa.feature.mfcc(y=audio, sr=sr, n_mfcc=40).T,axis=0)\n", "# chroma\n", "chroma_l = librosa.feature.chroma_stft(S=stft, sr=sr)\n", "chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sr).T,axis=0)\n", "# mel\n", "mel_l = librosa.feature.melspectrogram(audio, sr=sr)\n", "mel = np.mean(librosa.feature.melspectrogram(audio, sr=sr).T,axis=0)\n", "# contrast\n", "contrast_l = librosa.feature.spectral_contrast(S=stft, sr=sr)\n", "contrast = np.mean(librosa.feature.spectral_contrast(S=stft, sr=sr).T,axis=0)\n", "# tonnetz\n", "tonnetz_l = librosa.feature.tonnetz(y=librosa.effects.harmonic(audio), sr=sr)\n", "tonnetz = np.mean(librosa.feature.tonnetz(y=librosa.effects.harmonic(audio), sr=sr).T,axis=0)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "cda30f2a6b26262c9089794b664714fb621b9570", "_cell_guid": "b226a99e-4fdd-473a-ae0d-ca127401f60c"}, "source": ["print(stft.shape)\n", "print('mfcc')\n", "print(mfcc_l.shape)\n", "print(mfccs.shape)\n", "print('chroma')\n", "print(chroma_l.shape)\n", "print(chroma.shape)\n", "print('mel')\n", "print(mel_l.shape)\n", "print(mel.shape)\n", "print('contrast')\n", "print(contrast_l.shape)\n", "print(contrast.shape)\n", "print('tonnetz')\n", "print(tonnetz_l.shape)\n", "print(tonnetz.shape)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "2a1bed0aab68b8867d6552bc387c313ff5554816", "_cell_guid": "6d025255-8000-4550-9da9-70143ebaba72"}, "source": ["plt.imshow(mfcc_l)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "ab88356df7f22c1328de6b3d70dc479ca8e09885", "_cell_guid": "bccbf896-b170-491f-8cc5-18869d8965a4"}, "source": ["plt.imshow(chroma_l)"], "execution_count": null, "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true, "_uuid": "e97d3b60db8f6546e68246db5ecda861665c615c", "_cell_guid": "81ae953c-b693-45da-a33a-c95c8c5ac6cb"}, "source": [], "execution_count": null, "outputs": []}], "metadata": {"kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}, "language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "name": "python", "file_extension": ".py", "nbconvert_exporter": "python", "version": "3.6.3", "pygments_lexer": "ipython3", "mimetype": "text/x-python"}}, "nbformat": 4}