{"cells":[{"metadata":{"_cell_guid":"d524ad8f-d024-4ba3-8bbc-2435ec3d0dba","_uuid":"7bb30a6f2a18be45a491092a6e2dfcdcce71a2a0"},"cell_type":"markdown","source":"## Preface\nThis notebooks aims to build a CNN + LSTM model.\n\nIt uses specgrams of wav files(rate 16000) as inputs.\n\n\n## File Structure\nThis script assumes data are stored in following strcuture:\n\nspeech\n\n├── test            \n\n│   └── audio #test wavfiles\n\n├── train           \n\n│   ├── audio #train wavfiles\n\n└── model #store models\n\n│\n\n└── out #store sub.csv\n\n## Improve This Script\nHere are some ways to improve it's performance.\n1. Use audio data augmentation techniques.\n2. Create more 'silence' wav files using chop_audio.\n3. Build deeper CNN or  RNN.\n4. Train for longer epochs\n\n## After Words\nAfter the submission the score I'm getting is 0.74, feedback is welcome.\n\nFeel free to share your ideas in the comment sections."},{"metadata":{"_cell_guid":"f7fd8bcb-4451-4d47-bfe8-491c94b3b4eb","_uuid":"712710f20b00f97271136cfeab9937a4c6a2458b","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy.io import wavfile\nfrom scipy import signal\nfrom glob import glob\nimport re\nimport pandas as pd\nimport gc\nfrom scipy.io import wavfile\n\nfrom keras import optimizers, losses, activations, models\nfrom keras.layers import Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization\nfrom sklearn.model_selection import train_test_split\nimport keras","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fb35a2f1-9301-4693-a9ef-9d180b630f05","_uuid":"4b1ba61998e14e15c822c605dbe5961bfed36014"},"cell_type":"markdown","source":"The original sample rate is 16000, and we will keep it the same."},{"metadata":{"_cell_guid":"dc66e1df-f1eb-4df4-ba1a-65b9f1675953","_uuid":"4cc586519523b28d1d595716d8709ace9f27ac9c","trusted":true},"cell_type":"code","source":"L = 16000\nlegal_labels = 'yes no up down left right on off stop go silence unknown'.split()\n\n#src folders\nroot_path = r'..'\nout_path = r'.'\nmodel_path = r'.'\ntrain_data_path = os.path.join(root_path, 'input', 'train', 'audio')\ntest_data_path = os.path.join(root_path, 'input', 'test', 'audio')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e53561e4-1c98-44c0-9245-d87f7957faa5","_uuid":"d9a08781f22e574bb1eb0dc29adeb8dddebc8b51"},"cell_type":"markdown","source":"Here is the log_specgram function."},{"metadata":{"_cell_guid":"0fd0b579-8b6f-4253-bf3a-7f75115a42d6","_uuid":"e7ea2c277b6459e532721452ec3cd80d585eae1e","trusted":true},"cell_type":"code","source":"def log_specgram(audio, sample_rate, window_size=20,\n                 step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    freqs, times, spec = signal.spectrogram(audio,\n                                    fs=sample_rate,\n                                    window='hann',\n                                    nperseg=nperseg,\n                                    noverlap=noverlap,\n                                    detrend=False)\n    return freqs, times, np.log(spec.T.astype(np.float32) + eps)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c54cda36-777e-4129-bac1-af2d1ed2706e","_uuid":"5a04e71fe7e66e1a31835feebdfef4c63920faf8"},"cell_type":"markdown","source":"Following is the utility function to grab all wav files inside train data folder."},{"metadata":{"_cell_guid":"956f3150-544d-46ed-b0ec-da1c1fb142b4","_uuid":"964d71a229e9d4560b9118fa1c80804ebf8d6be8","trusted":true},"cell_type":"code","source":"def list_wavs_fname(dirpath, ext='wav'):\n    print(dirpath)\n    fpaths = glob(os.path.join(dirpath, r'*/*' + ext))\n    pat = r'.+/(\\w+)/\\w+\\.' + ext + '$'\n    labels = []\n    for fpath in fpaths:\n        r = re.match(pat, fpath)\n        if r:\n            labels.append(r.group(1))\n    pat = r'.+/(\\w+\\.' + ext + ')$'\n    fnames = []\n    for fpath in fpaths:\n        r = re.match(pat, fpath)\n        if r:\n            fnames.append(r.group(1))\n    return labels, fnames","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"41025a55-8497-43cf-b316-003af7d9d19f","_uuid":"fc18e87793888952e81a867dd95b1dcc455f9932"},"cell_type":"markdown","source":"__pad_audio__ will pad audios that are less than 16000(1 second) with 0s to make them all have the same length.\n\n__chop_audio__ will chop audios that are larger than 16000(eg. wav files in background noises folder) to 16000 in length. In addition, it will create several chunks out of one large wav files given the parameter 'num'.\n\n__label_transform__ transform labels into dummies values. It's used in combination with softmax to predict the label."},{"metadata":{"_cell_guid":"200c34a1-851a-4447-9ff7-b4e541f090c6","_uuid":"94e40aef3899acfd3ed85557caa66fee5dd47db2","trusted":true},"cell_type":"code","source":"def pad_audio(samples):\n    if len(samples) >= L: return samples\n    else: return np.pad(samples, pad_width=(L - len(samples), 0), mode='constant', constant_values=(0, 0))\n\ndef chop_audio(samples, L=16000, num=20):\n    for i in range(num):\n        beg = np.random.randint(0, len(samples) - L)\n        yield samples[beg: beg + L]\n\ndef label_transform(labels):\n    nlabels = []\n    for label in labels:\n        if label == '_background_noise_':\n            nlabels.append('silence')\n        elif label not in legal_labels:\n            nlabels.append('unknown')\n        else:\n            nlabels.append(label)\n    return pd.get_dummies(pd.Series(nlabels))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"dae2a45a-f7ab-4e84-bc73-688eda6eca8e","_uuid":"267314ef41c459c8b6ab903d721980fdd62b4106"},"cell_type":"markdown","source":"Next, we use functions declared above to generate x_train and y_train.\nlabel_index is the index used by pandas to create dummy values, we need to save it for later use."},{"metadata":{"_cell_guid":"4c8d9fdf-ea3e-45fa-b7ef-52542c70b9db","_uuid":"81bc9722dfb036c73721ae44829d429489662e75","trusted":true},"cell_type":"code","source":"labels, fnames = list_wavs_fname(train_data_path)\n\nnew_sample_rate = 16000\ny_train = []\nx_train = []\n\nfor label, fname in zip(labels, fnames):\n    sample_rate, samples = wavfile.read(os.path.join(train_data_path, label, fname))\n    samples = pad_audio(samples)\n    if len(samples) > 16000:\n        n_samples = chop_audio(samples)\n    else: n_samples = [samples]\n    for samples in n_samples:\n        resampled = signal.resample(samples, int(new_sample_rate / sample_rate * samples.shape[0]))\n        _, _, specgram = log_specgram(resampled, sample_rate=new_sample_rate)\n        y_train.append(label)\n        x_train.append(specgram)\nx_train = np.array(x_train)\n# x_train = x_train.reshape(tuple(list(x_train.shape) + [1]))\ny_train = label_transform(y_train)\nlabel_index = y_train.columns.values\ny_train = y_train.values\ny_train = np.array(y_train)\ndel labels, fnames\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"56921cf3-1269-4b29-876d-abdd31eb150a","_uuid":"a87a77b76c42da61ca0bec395c71bef795a9e928"},"cell_type":"markdown","source":"RNN declared below.\nThe specgram created will be of shape (99, 161), but in order to fit into Conv2D layer, we need to reshape it."},{"metadata":{"_cell_guid":"b97e8887-b593-4d88-95c8-fc8f1dd5ca72","_uuid":"60af394ad8e91fb868ea32dbb6ac6a725b5935c9","trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\nfrom tensorflow.keras.layers import Dense, SimpleRNN, LSTM, Bidirectional, TimeDistributed, Conv1D, ZeroPadding1D, GRU\nfrom tensorflow.keras.layers import Lambda, Input, Dropout, Masking, BatchNormalization, Activation\nfrom tensorflow.keras.models import Model\n\ndef cnn_lstm(input_dim, output_dim, dropout=0.2, n_layers=1):\n\n#     # Input data type\n    dtype = 'float32'\n\n    # ---- Network model ----\n    input_data = Input(name='the_input', shape=input_dim, dtype=dtype)\n\n    # 1 x 1D convolutional layers with strides 4\n    x = Conv1D(filters=256, kernel_size=10, strides=4, name='conv_1')(input_data)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Dropout(dropout, name='dropout_1')(x)\n        \n    x = LSTM(128, activation='relu', return_sequences=True,\n             dropout=dropout, name='lstm_1')(x)\n    x = LSTM(128, activation='relu', return_sequences=False,\n             dropout=dropout, name='lstm_2')(x)\n\n#     # 1 fully connected layer DNN ReLu with default 20% dropout\n    x = Dense(units=64, activation='relu', name='fc')(x)\n    x = Dropout(dropout, name='dropout_2')(x)\n\n    # Output layer with softmax\n    y_pred = Dense(units=output_dim, activation='softmax', name='softmax')(x)\n\n    network_model = Model(inputs=input_data, outputs=y_pred)\n    \n    return network_model\n\ninput_dim = (99, 161)\nclasses = len(legal_labels)\nK.clear_session()\nmodel = cnn_lstm(input_dim, classes)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam, SGD\nfrom tensorflow.keras.callbacks import TensorBoard\n\nsgd = SGD(lr=0.00001, clipnorm=1.0)\nadam = Adam(lr=1e-4, clipnorm=1.0)\n\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=adam,\n              metrics=['accuracy'])\nhistory = model.fit(x_train, y_train,\n                    batch_size=128, epochs=10,\n#                     validation_data=(X_val, Y_val),\n                    callbacks=[TensorBoard(log_dir='logs',\n                                           histogram_freq=1,\n                                           update_freq='epoch')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(os.path.join(model_path, 'rnn.model'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint(os.getcwd())\nprint(os.listdir())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import load_model\n\nmodel = load_model('rnn.model')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"060811f5-34ac-4fc2-92ba-c4276606c2a0","_uuid":"2e3fa8d9706f47e69d0b74afcb68280f8a5de706"},"cell_type":"markdown","source":"Test data is way too large to fit in RAM, we need to process them one by one.\nGenerator test_data_generator will create batches of test wav files to feed into CNN."},{"metadata":{"_cell_guid":"7dfe0801-a636-4123-8367-ab2f19c97800","_uuid":"646b6bcfbde7eae53cd8822b8838c575859e51ce","trusted":true},"cell_type":"code","source":"def test_data_generator(batch=16):\n    fpaths = glob(os.path.join(test_data_path, '*wav'))\n    i = 0\n    for path in fpaths:\n        if i == 0:\n            imgs = []\n            fnames = []\n        i += 1\n        rate, samples = wavfile.read(path)\n        samples = pad_audio(samples)\n        resampled = signal.resample(samples, int(new_sample_rate / rate * samples.shape[0]))\n        _, _, specgram = log_specgram(resampled, sample_rate=new_sample_rate)\n        imgs.append(specgram)\n        fnames.append(path.split('\\\\')[-1])\n        if i == batch:\n            i = 0\n            imgs = np.array(imgs)\n            imgs = imgs.reshape(tuple(list(imgs.shape) + [1]))\n            yield fnames, imgs\n    if i < batch:\n        imgs = np.array(imgs)\n        imgs = imgs.reshape(tuple(list(imgs.shape) + [1]))\n        yield fnames, imgs\n    raise StopIteration()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"22992a27-deda-4a35-b34c-4aa87ad173ec","_uuid":"c6d2516a6d5bd3c6a108d7e28565edaa65830958"},"cell_type":"markdown","source":"We use the trained model to predict the test data's labels.\nHowever, since Kaggle doesn't provide test data, the following sections won't be executed here."},{"metadata":{"_cell_guid":"7fa8feb8-236e-46c5-8432-014f7e27484d","_uuid":"56194039cac16f5d86a322e67641cbeafda9857d","trusted":true},"cell_type":"code","source":"# exit() #delete this\ndel x_train, y_train\ngc.collect()\n\nindex = []\nresults = []\nfor fnames, imgs in test_data_generator(batch=32):\n    predicts = model.predict(imgs)\n    predicts = np.argmax(predicts, axis=1)\n    predicts = [label_index[p] for p in predicts]\n    index.extend(fnames)\n    results.extend(predicts)\n\ndf = pd.DataFrame(columns=['fname', 'label'])\ndf['fname'] = index\ndf['label'] = results\ndf.to_csv(os.path.join(out_path, 'sub.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","language":"python","display_name":"Python 3"},"language_info":{"nbconvert_exporter":"python","name":"python","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","file_extension":".py","version":"3.6.3"}},"nbformat":4,"nbformat_minor":1}