{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **MIDAS Task 1**\n\nFor this task, two different methods are tested and evaluated. The methods differ mainly on the audio feature used for classification. They are listed below as:\n1. Using Normalized Log Spectrogram of Audio Samples\n2. Using MFCC's and their higher order Deltas\n\n**This notebook only has the first method, which gavebest results. The other method is in the other experiments, placed in the experiments folder. This notebook has:**\n* Libraries Required\n* Data Unzip\n* Data Check\n* EDA\n* Data Loading, Preprocessing, Feature Extraction\n* Training andTesting\n\nReferences:\n\n1. David S., Speech representation and Data Exploration: Has been used to better understand the data","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom tqdm import tqdm\nfrom glob import glob\nimport os\nimport random\nfrom os.path import isdir, join\nimport regex as re\nimport gc\nimport pickle\n\nimport librosa\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nfrom sklearn.decomposition import PCA\n\nimport matplotlib.pyplot as plt\nimport librosa.display\nimport IPython.display as ipd \n\nfrom keras import optimizers, losses, activations, models\nfrom keras.layers import Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization\nfrom sklearn.model_selection import train_test_split\nimport keras\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Unzip","metadata":{}},{"cell_type":"code","source":"!apt-get install -y p7zip-full\n!7z x ../input/tensorflow-speech-recognition-challenge/train.7z\n!7z x ../input/tensorflow-speech-recognition-challenge/test.7z","metadata":{"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nCheck Training Audio Folder\n'''\n\ntrain_audio_path = 'train/audio/'\nprint(len(os.listdir(train_audio_path)))\nprint(os.listdir(train_audio_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"'''\nLoading a sample file to study\n'''\n\n# At 8000 \nfilename = '/yes/0a7c2a8d_nohash_0.wav'\nsamples2, sample_rate2 = librosa.load(str(train_audio_path) + filename, 8000)\n\n# At 16000 \nfilename = '/yes/0a7c2a8d_nohash_0.wav'\nsamples, sample_rate = librosa.load(str(train_audio_path) + filename, 16000)\nprint('Sample Rate: ', sample_rate)\nprint()\nipd.Audio(samples, rate=sample_rate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nHELPER FUNTIONS FOR PLOTTING AND DATA ANALYSIS\n'''\n\n\n'''\nDefining a funtion to calculate LOG SPECTROGRAM\nModule used -> SCIPY SIGNAL\n\nParameters:\n\nwindow - Hamming window is used (Trend observed in multiple papers)\nfs - Sample Rate (8000)\nnperseg - Length of each sample segment\nnoverlap - Length of overlap of windows\n\nNote that the spectrogram will also be normalized over its mean and standard deviation later.\n'''\n\ndef log_spectrogram (audio, sample_rate, window_size=20, step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    \n    freqs, times, spec = signal.spectrogram(audio,\n                                    fs=sample_rate,\n                                    window='hamming',\n                                    nperseg=nperseg,\n                                    noverlap=noverlap,\n                                    detrend=False)\n    \n    return freqs, times, np.log(spec.T.astype(np.float32) + eps)\n\n\n\n'''\nDefining a funtion to calculate FFT\nReference - David S., Speech representation and Data Exploration\n\nModule -Scipy FFT\n\nNotes:\n1. FFT is simmetrical, so we take just the first half\n2. FFT is also complex, to we take just the real part (abs)\n'''\n\ndef fft_self(y, fs):\n    T = 1.0 / fs\n    N = y.shape[0]\n    yf = fft(y)\n    xf = np.linspace(0.0, 1.0/(2.0*T), N//2)\n    vals = 2.0/N * np.abs(yf[0:N//2]) \n    return xf, vals","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Plotting Spectrogram of Single Sample","metadata":{}},{"cell_type":"code","source":"freqs, times, spectrogram = log_spectrogram(samples, sample_rate)\n\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + filename)\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)\n\nax2 = fig.add_subplot(212)\nax2.imshow(spectrogram.T, aspect='auto', origin='lower', \n           extent=[times.min(), times.max(), freqs.min(), freqs.max()])\nax2.set_yticks(freqs[::16])\nax2.set_xticks(times[::16])\nax2.set_title('Spectrogram of ' + filename)\nax2.set_ylabel('Freqs in Hz')\nax2.set_xlabel('Seconds')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Plotting MFCCs of Single Sample","metadata":{}},{"cell_type":"code","source":"'''\nUsing Librosa to plot mel scaled power spectrogram and MFCCs of signals\n\n1. First the Power Spectrogram is shown.\n2. Then the MFCC of the same is shown.\n'''\n\nS = librosa.feature.melspectrogram(samples, sr=sample_rate, n_mels=128)\n\n# Convert to log scale (dB).\nlog_S = librosa.power_to_db(S, ref=np.max)\n\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(log_S, sr=sample_rate, x_axis='time', y_axis='mel')\nplt.title('Mel power spectrogram ')\nplt.colorbar(format='%+02.0f dB')\nplt.tight_layout()\n\nmfcc = librosa.feature.mfcc(S=log_S, n_mfcc=13)\n\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(mfcc)\nplt.ylabel('MFCC coeffs')\nplt.xlabel('Time')\nplt.title('MFCC')\nplt.colorbar()\nplt.tight_layout()\n\n'''\nAlso Plotting Second Order Delta MFCCs\n'''\n\nmfcc = librosa.feature.mfcc(S=log_S, n_mfcc=13)\ndelta2_mfcc = librosa.feature.delta(mfcc, order=2)\n\nplt.figure(figsize=(12, 4))\nlibrosa.display.specshow(delta2_mfcc)\nplt.ylabel('MFCC coeffs')\nplt.xlabel('Time')\nplt.title('MFCC Second Order Delta')\nplt.colorbar()\nplt.tight_layout()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### FFTs at 16000 and 8000 Sampling Rate","metadata":{}},{"cell_type":"code","source":"'''\nSampling Rate - 16000 Hz\n'''\n\nxf, vals = fft_self(samples, sample_rate)\nplt.figure(figsize=(12, 4))\nplt.title('FFT of recording sampled with ' + str(sample_rate) + ' Hz')\nplt.plot(xf, vals)\nplt.xlabel('Frequency')\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nSampling Rate - 8000 Hz\n'''\n\nxf, vals = fft_self(samples2, sample_rate2)\nplt.figure(figsize=(12, 4))\nplt.title('FFT of recording sampled with ' + str(sample_rate2) + ' Hz')\nplt.plot(xf, vals)\nplt.xlabel('Frequency')\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Mean FFTs and Spectrograms of all classes\n\n*Reference - David S., Speech representation and Data Exploration*","metadata":{}},{"cell_type":"code","source":"dirs = [f for f in os.listdir(train_audio_path) if isdir(join(train_audio_path, f))]\nto_keep = 'yes no up down left right on off stop go'.split()\ndirs = [d for d in dirs if d in to_keep]\n\nprint(dirs)\n\nfor direct in dirs:\n    vals_all = []\n    spec_all = []\n\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(train_audio_path + direct + '/' + wav)\n        if samples.shape[0] != 16000:\n            continue\n        xf, vals = fft_self(samples, 16000)\n        vals_all.append(vals)\n        freqs, times, spec = log_spectrogram(samples, 16000)\n        spec_all.append(spec)\n\n    plt.figure(figsize=(14, 4))\n    plt.subplot(121)\n    plt.title('Mean fft of ' + direct)\n    plt.plot(np.mean(np.array(vals_all), axis=0))\n    plt.grid()\n    plt.subplot(122)\n    plt.title('Mean specgram of ' + direct)\n    plt.imshow(np.mean(np.array(spec_all), axis=0).T, aspect='auto', origin='lower', \n               extent=[times.min(), times.max(), freqs.min(), freqs.max()])\n    plt.yticks(freqs[::16])\n    plt.xticks(times[::16])\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Loading, Preprocessing and Feature Extraction\n\n##### Details:\n\n1. All files are sampled at 8000 HZ\n2. The file length has been kept constant at 1 second.\n2. Zero Padding has been used in sound processing to make audio files of same length. The same has been implemented here.\n3. For extremely long audio files, they have been split into smaller 1 second samples.\n\n##### Features\n\n1. Log Spectrograms are loaded for each audio sample.\n2. These are all of the same shape since data is padded to same size.\n3. They are normalized over the entire dataset.","metadata":{}},{"cell_type":"code","source":"'''\nVariables to load data.\n\nsr - Sample Rate\nLegal Labels - Categories to Classify\n'''\n\nsr = 16000\nlegal_labels = 'yes no up down left right on off stop go silence unknown'.split()\n\n# Paths\ntrain_data_path = 'train/audio'\ntest_data_path = 'test'\n\n# Path Helper\next = 'wav'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nUtility Function\n\nTask - Collect all file names.\n'''\n\ndef collect_files(path):\n    \n    # Simple Regex to collect paths\n    fpaths = glob(os.path.join(path, r'*/*' + ext))\n    pat = r'.+/(\\w+)/\\w+\\.' + ext + '$'\n    \n    labels = []\n    for fpath in fpaths:\n        fpath = fpath.replace('\\\\', '/')\n        r = re.match(pat, fpath)\n        if r:\n            labels.append(r.group(1))\n    pat = r'.+/(\\w+\\.' + ext + ')$'\n    fnames = []\n    for fpath in fpaths:\n        fpath = fpath.replace('\\\\', '/')\n        r = re.match(pat, fpath)\n        if r:\n            fnames.append(r.group(1))\n    return labels, fnames","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nPadding -> Pad smaller samples with 0\n\nReduce -> Reduce audio to predefined size.\n\nLabellize -> Tranform label to suitable format for model.\n'''\n\ndef padding(samples):\n    if len(samples) >= sr: \n        return samples\n    else: \n        return np.pad(samples, pad_width=(sr - len(samples), 0), mode='constant', constant_values=(0, 0))\n\ndef reduce(samples, sr=16000, num=20):\n    for i in range(num):\n        cut = np.random.randint(0, len(samples) - sr)\n        yield samples[cut: cut + sr]\n\ndef labellize(labels):\n    nlabels = []\n    for label in labels:\n        if label == '_background_noise_':\n            nlabels.append('silence')\n        elif label not in legal_labels:\n            nlabels.append('unknown')\n        else:\n            nlabels.append(label)\n    return pd.get_dummies(pd.Series(nlabels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nGet paths and labels.\n'''\n\nlabels, filenames = collect_files(train_data_path)\n\nY_tr = []\nX_tr = []\n\n'''\nLoad Log Specs\n'''\n\nfor label, fname in tqdm(zip(labels, filenames)):\n    \n    sample_rate, samples = wavfile.read(os.path.join(train_data_path, label, fname))\n    samples = padding(samples)\n    if len(samples) > 16000:\n        n_samples = reduce(samples)\n    else: \n        n_samples = [samples]\n    for samples in n_samples:\n        _, _, specgram = log_spectrogram(samples, sample_rate=sr)\n        Y_tr.append(label)\n        X_tr.append(specgram)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nConverting to Numpy Arrays and labellizing.\n'''\n\nX_train = np.array(X_tr)\nX_train = X_train.reshape(tuple(list(X_train.shape) + [1]))\nY_train = labellize(Y_tr)\nlabel_index = Y_train.columns.values\nY_train = Y_train.values\nY_train = np.array(Y_train)\ndel labels, filenames, X_tr, Y_tr\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"markdown","source":"* Batch Normalization used to normalize all spectrogram.\n* Max Pooling and Dropout used to prevent over-fitting.\n* Simple Convolution Layer Used. \n* A heavy model was not used since RAM was limited to 16GB on kaggle.","metadata":{}},{"cell_type":"code","source":"'''\nDefine Params\n'''\n\ninput_shape = (99, 161, 1)\nnclass = 12\ninp = Input(shape=input_shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nBuild Model\n'''\n\nnorm_inp = BatchNormalization()(inp)\nimg_1 = Convolution2D(8, kernel_size=2, activation=activations.relu)(norm_inp)\nimg_1 = Convolution2D(8, kernel_size=2, activation=activations.relu)(norm_inp)\nimg_1 = MaxPooling2D(pool_size=(2, 2))(img_1)\nimg_1 = Dropout(rate=0.2)(img_1)\nimg_1 = Convolution2D(16, kernel_size=2, activation=activations.relu)(img_1)\nimg_1 = Convolution2D(16, kernel_size=2, activation=activations.relu)(img_1)\nimg_1 = MaxPooling2D(pool_size=(2, 2))(img_1)\nimg_1 = Dropout(rate=0.2)(img_1)\nimg_1 = Convolution2D(32, kernel_size=3, activation=activations.relu)(img_1)\nimg_1 = Convolution2D(32, kernel_size=5, activation=activations.relu)(img_1)\nimg_1 = MaxPooling2D(pool_size=(2, 2))(img_1)\nimg_1 = Dropout(rate=0.2)(img_1)\nimg_1 = Flatten()(img_1)\n\ndense_1 = BatchNormalization()(Dense(32, activation=activations.relu)(img_1))\ndense_1 = BatchNormalization()(Dense(64, activation=activations.relu)(dense_1))\ndense_1 = Dense(nclass, activation=activations.softmax)(dense_1)\n\nmodel = models.Model(inputs=inp, outputs=dense_1)\nopt = optimizers.Adam()\n\nmodel.compile(optimizer=opt, loss=losses.binary_crossentropy)\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nSplit into train and validation and run.\n'''\n\nX_train, X_valid, Y_train, Y_valid = train_test_split(X_train, Y_train, test_size=0.15, random_state=1)\nmodel.fit(X_train, Y_train, batch_size=16, validation_data=(X_valid, Y_valid), epochs=5, shuffle=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('cnn.model')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing","metadata":{}},{"cell_type":"code","source":"'''\nCreae generator for test data. Since Data is too large.\n'''\n\ntest_data_path = 'test/audio'\n\ndef test_data_generator(batch=16):\n    fpaths = glob(os.path.join(test_data_path, '*wav'))\n    i = 0\n    for path in fpaths:\n        if i == 0:\n            imgs = []\n            fnames = []\n        i += 1\n        rate, samples = wavfile.read(path)\n        samples = pad_audio(samples)\n        _, _, specgram = log_spectrogram(resampled, sample_rate=new_sample_rate)\n        imgs.append(specgram)\n        fnames.append(path.split('\\\\')[-1])\n        if i == batch:\n            i = 0\n            imgs = np.array(imgs)\n            imgs = imgs.reshape(tuple(list(imgs.shape) + [1]))\n            yield fnames, imgs\n    if i < batch:\n        imgs = np.array(imgs)\n        imgs = imgs.reshape(tuple(list(imgs.shape) + [1]))\n        yield fnames, imgs\n    raise StopIteration()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}