{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# Optimized Keras Sequential Conv1D version"},{"metadata":{"trusted":true},"cell_type":"code","source":"# io\nimport os\nfrom os.path import isdir, join\nfrom pathlib import Path\nimport pandas as pd\n\n# Scientific Math\nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport plotly.offline as py\nimport plotly.graph_objs as go\n\n# Audio\nimport IPython.display as ipd\n\n# Deep learning\nimport tensorflow.keras as keras\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten\nfrom tensorflow.keras import Input, layers\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.models import load_model\n\nimport random\nimport copy\nimport librosa\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../input/tensorflow-speech-recognition-challenge/'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_audio_path = '../input/tensorflow-speech-recognition-challenge/train/audio/'\nprint(os.listdir(train_audio_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load Data\ndirs = [f for f in os.listdir(train_audio_path) if isdir(join(train_audio_path, f))]\ndirs.sort()\nprint('Number of labels: ' + str(len(dirs[1:])))\nprint(dirs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_wav = []\nunknow_wav = []\nlabel_all = []\nlabel_value = {}\ntarget_list = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']\nunknow_list = set([d for d in dirs if d not in target_list and d != '_background_noise_'])\nprint('target_list: ', end='')\nprint(target_list)\nprint('unknowns_list: ', end='')\nprint(unknow_list)\nprint('silence: _background_noise_')\nbackground = [f for f in os.listdir(join(train_audio_path, '_background_noise_')) if f.endswith('.wav')]\nbackground_noise = []\nfor wav in background:\n    samples, sample_rate = librosa.load(join(train_audio_path, '_background_noise_', wav))\n    samples = librosa.resample(samples, sample_rate, 8000)\n    background_noise.append(samples)\nfor i,d in enumerate(dirs[1:]):\n    waves = [f for f in os.listdir(join(train_audio_path, d)) if f.endswith('.wav')]\n    label_value[d] = i\n    print(str(i)+':'+str(d)+' ', end='')\n    for wav in waves:\n        samples, sample_rate = librosa.load(join(train_audio_path, d, wav), sr=16000)\n        samples = librosa.resample(samples, sample_rate, 8000)\n        if len(samples) != 8000: continue\n        if d in unknow_list:\n            unknow_wav.append(samples)\n        else:\n            label_all.append(d)\n            all_wav.append([samples, d])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split wav, label\nwav_all = np.reshape(np.delete(all_wav, 1, 1), (len(all_wav)))\nlabel_all = [i for i in np.delete(all_wav, 0, 1).tolist()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data Augmentation, 10% amplitude from 8000 samples in 1 sec noise\ndef get_one_noise(noise_num=0):\n    selected_noise = background_noise[noise_num]\n    start_idx = random.randint(0, len(selected_noise)-1-8000)\n    return selected_noise[start_idx:(start_idx + 8000)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_ratio = 0.1\nnoised_wav = []\naugment = 1\ndelete_index = []\nfor i in range(augment):\n    new_wav = []\n    noise = get_one_noise(i)\n    for i, s in enumerate(wav_all):\n        if len(s) != 8000:\n            delete_index.append(i)\n            continue\n        s += max_ratio * noise\n        noised_wav.append(s)\nnp.delete(wav_all, delete_index)\nnp.delete(label_all, delete_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wav_vals = np.array([x for x in wav_all])\nlabel_vals = [x for x in label_all]\nwav_vals.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = copy.deepcopy(label_vals)\nfor _ in range(augment):\n    label_vals = np.concatenate((label_vals, labels), axis=0)\nlabel_vals = label_vals.reshape(-1, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Random sampling from unknown wav data\nunknown = unknow_wav\naugment_unknown = 2\nnp.random.shuffle(unknow_wav)\nunknown = np.array(unknown)\nunknown = unknown[:2000*(augment_unknown+1)]\nunknown_label = np.array(['unknown' for _ in range(2000*(augment_unknown+1))])\nunknown_label = unknown_label.reshape(2000*(augment_unknown+1), 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# some data may have different length, so delete them\ndelete_index = []\nfor i, w in enumerate(unknown):\n    if len(w) != 8000:\n        delete_index.append(i)\nunknown = np.delete(unknown, delete_index, axis=0)\nunknown_label = np.delete(unknown_label, delete_index, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Random samping from 'background_noise'\nsilence_wav = []\naugment_silence = 1\nnum_wav = (2000*(augment_silence+1)) // len(background_noise)\nfor i, _ in enumerate(background_noise):\n    for _ in range((2000*(augment_silence+1))//len(background_noise)):\n        silence_wav.append(get_one_noise(i))\nsilence_wav = np.array(silence_wav)\nsilence_label = np.array(['silence' for _ in range(num_wav*len(background_noise))])\nsilence_label = silence_label.reshape(-1, 1)\nsilence_wav.shape, silence_label.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shape_ = (-1, 8000)\nwav_vals = np.reshape(wav_vals, shape_)\nnoised_wav = np.reshape(noised_wav, shape_)\nunknown = np.reshape(unknown, shape_)\nsilence_wav = np.reshape(silence_wav, shape_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check dimensions\nprint(wav_vals.shape)\nprint(noised_wav.shape)\nprint(unknown.shape)\nprint(silence_wav.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(label_vals.shape)\nprint(unknown_label.shape)\nprint(silence_label.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# concatenate wavs, labels\nwav_vals = np.concatenate((wav_vals, noised_wav), axis=0)\nwav_vals = np.concatenate((wav_vals, unknown), axis=0)\nwav_vals = np.concatenate((wav_vals, silence_wav), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_vals = np.concatenate((label_vals, unknown_label), axis=0)\nlabel_vals = np.concatenate((label_vals, silence_label), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(wav_vals), len(label_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare train and validation data\n# train_wav, test_wav, train_label, test_label = train_test_split(wav_vals, label_vals,\\\n#     test_size=0.2, random_state=1993, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare train data\ntrain_wav, train_label = shuffle(wav_vals, label_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_wav))\n# print(len(test_wav))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Hyper-parameters\nlr = .001\ngenerations = 20000\nnum_gens_to_wait = 250\nbatch_size = 512\ndrop_out_rate = .5\ninput_shape = (8000, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_wav.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For Conv1D and Channel\ntrain_wav = train_wav.reshape(-1, 8000, 1)\n# test_wav = test_wav.reshape(-1, 8000, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_wav.shape)\n# print(test_wav.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_value = target_list\nlabel_value.append('unknown')\nlabel_value.append('silence')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_label_value = {}\nfor i, la in enumerate(label_value):\n    new_label_value[la] = i\nlabel_value = new_label_value\nlabel_num2word = {v:k for k, v in label_value.items()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_value, label_num2word","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make label data 'string' to class num\ntemp = []\nfor v in train_label:\n    temp.append(label_value[v[0]])\ntrain_label = np.array(temp)\n# temp = []\n# for v in test_label:\n#     temp.append(label_value[v[0]])\n# test_label = np.array(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make label data one hot vector\ntrain_label = keras.utils.to_categorical(train_label, len(label_value))\n# test_label = keras.utils.to_categorical(test_label, len(label_value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train_wav dimension: ' + str(train_wav.shape))\nprint('train_label dimension: ' + str(train_label.shape))\n# print('test_wav dimension: ' + str(test_wav.shape))\n# print('test_label dimension: ' + str(test_label.shape))\nprint('number of labels: ' + str(len(label_value)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Conv1D Model\ninput_tensor = Input(shape=(input_shape))\nx = layers.Conv1D(8, 11, padding='valid', activation='relu', strides=1)(input_tensor)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(16, 7, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(32, 5, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(64, 5, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(128, 3, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Flatten()(x)\nx = layers.Dense(256, activation='relu')(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Dense(128, activation='relu')(x)\nx = layers.Dropout(drop_out_rate)(x)\noutput_tensor = layers.Dense(len(label_value), activation='softmax')(x)\n\nmodel = tf.keras.Model(input_tensor, output_tensor)\nmodel.compile(loss=keras.losses.categorical_crossentropy, \\\n    optimizer=keras.optimizers.Adam(lr=lr), metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train with validation set\n# history = model.fit(train_wav, train_label, validation_data=[test_wav, test_label],\n#     batch_size=batch_size, epochs=100, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train without validation set\nhistory = model.fit(train_wav, train_label, batch_size=batch_size, epochs=100, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot loss & accuracy for both train and validation set\n# plt.plot(history.history['acc'])\n# plt.plot(history.history['val_acc'])\n# plt.title('model accuracy')\n# plt.ylabel('accuracy')\n# plt.xlabel('epoch')\n# plt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# # summarize history for loss\n# plt.plot(history.history['loss'])\n# plt.plot(history.history['val_loss'])\n# plt.title('model loss')\n# plt.ylabel('loss')\n# plt.xlabel('epoch')\n# plt.legend(['train', 'test'], loc='upper left')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot loss & accuracy for train set\nplt.plot(history.history['acc'])\nplt.plot(history.history['loss'])\nplt.title('model acc/loss')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['accuracy', 'loss'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save model weights together with architecture\nmodel.save('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_idx = 888\n# res = model.predict(test_wav[test_idx].reshape((-1, 8000, 1)))\n# print(label_num2word[np.argmax(res)], label_num2word[np.argmax(test_label[test_idx])])\n# librosa.output.write_wav('x.wav', test_wav[test_idx], 8000)\n# sr, ss = wavfile.read('x.wav')\n# ipd.Audio(ss, rate=8000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y_test = np.argmax(test_label, axis=1) # Convert one-hot to index\n# y_pred = np.argmax(model.predict(test_wav), axis=1) # Convert one-hot to index\n# print(classification_report(Y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# https://www.kaggle.com/hemingwei/tensorflow-speech-recognition-public-test-set\n# !7z x ../input/tensorflow-speech-recognition-public-test-set/test.7z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict for test data and output result\n# test_path = './test/audio/'\n# df = pd.read_csv('../input/tensorflow-speech-recognition-challenge/sample_submission.csv')\n# # test_data = np.zeros((158538, 8000))\n# # for i in range(len(df)):\n# #     samples, sample_rate = librosa.load(join(test_path, df.loc[i]['fname']), sr=16000)\n# #     samples = librosa.resample(samples, sample_rate, 8000)\n# #     test_data[i] = samples\n# # test_data = test_data.reshape(-1, 8000, 1)\n# # result = model.predict(test_data)\n# for i in range(len(df)):\n#     samples, sample_rate = librosa.load(join(test_path, df.loc[i]['fname']), sr=16000)\n#     samples = librosa.resample(samples, sample_rate, 8000)\n#     result = model.predict(np.reshape(samples, (-1, 8000, 1)))\n#     df.loc[i]['label'] = label_num2word[np.argmax(result)]\n# df.to_csv('submission.cvs', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ls -lh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# download output file\n# from IPython.display import FileLink\n# FileLink('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bbb = load_model('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_wav[:1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bbb.predict(train_wav[:1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}