{"cells":[{"metadata":{"trusted":true,"_uuid":"6e7b6f6347f596f78b3a8fb88cb7a820541998b1"},"cell_type":"code","source":"#path\nimport os\nfrom os.path import isdir, join\nfrom pathlib import Path\n\n# Scientific Math \nimport numpy as np\nfrom scipy.fftpack import fft\nfrom scipy import signal\nfrom scipy.io import wavfile\nfrom sklearn.model_selection import train_test_split\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport plotly.offline as py\nimport plotly.graph_objs as go\n\n#Deep learning\nimport tensorflow.keras as keras\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten\nfrom tensorflow.keras import Input, layers\nfrom tensorflow.keras import backend as K\n\nimport random\nimport copy\nimport librosa\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb0bfbe942e8dc8e6bf4864b7ec66212734657bb"},"cell_type":"code","source":"print(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1979ce525c49d97564617560c48fe00e40707b17"},"cell_type":"code","source":"train_audio_path = '../input/train/audio/'\nprint(os.listdir(train_audio_path))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc717705d62e8a2a312a983f59ab0406c0e0329b"},"cell_type":"markdown","source":"### Load Data\n\ntarget list is ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']\nunknown list is other\nsilence will be made from '_background_noise_'\n\nTrain data's sampling rate is 16000Hz, but for making lower computation cost, Resample to 8000hz\n\nAfter training, test set also will resample to 8000Hz"},{"metadata":{"trusted":true,"_uuid":"baf56ce3fd56eb7c6dd4b5d088105a631654071e"},"cell_type":"code","source":"dirs = [f for f in os.listdir(train_audio_path) if isdir(join(train_audio_path, f))]\ndirs.sort()\nprint('Number of labels: ' + str(len(dirs[1:])))\nprint(dirs)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"cdeb5698a33395d714bd7ec249f25a668734bc7d"},"cell_type":"code","source":"all_wav = []\nunknown_wav = []\nlabel_all = []\nlabel_value = {}\ntarget_list = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']\nunknown_list = [d for d in dirs if d not in target_list and d != '_background_noise_' ]\nprint('target_list : ',end='')\nprint(target_list)\nprint('unknowns_list : ', end='')\nprint(unknown_list)\nprint('silence : _background_noise_')\ni=0;\nbackground = [f for f in os.listdir(join(train_audio_path, '_background_noise_')) if f.endswith('.wav')]\nbackground_noise = []\nfor wav in background : \n    samples, sample_rate = librosa.load(join(join(train_audio_path,'_background_noise_'),wav))\n    samples = librosa.resample(samples, sample_rate, 8000)\n    background_noise.append(samples)\n\nfor direct in dirs[1:]:\n    waves = [f for f in os.listdir(join(train_audio_path, direct)) if f.endswith('.wav')]\n    label_value[direct] = i\n    i = i + 1\n    print(str(i)+\":\" +str(direct) + \" \", end=\"\")\n    for wav in waves:\n        samples, sample_rate = librosa.load(join(join(train_audio_path,direct),wav), sr = 16000)\n        samples = librosa.resample(samples, sample_rate, 8000)\n        if len(samples) != 8000 : \n            continue\n            \n        if direct in unknown_list:\n            unknown_wav.append(samples)\n        else:\n            label_all.append(direct)\n            all_wav.append([samples, direct])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9efa60ca094eeae08d34e078d4ec62231e50054"},"cell_type":"markdown","source":"split wav, label"},{"metadata":{"trusted":true,"_uuid":"8501a2e50843829e3a51853c683f105b040e954f"},"cell_type":"code","source":"wav_all = np.reshape(np.delete(all_wav,1,1),(len(all_wav)))\nlabel_all = [i for i in np.delete(all_wav,0,1).tolist()]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3239651e24e6decb63e5c5c9221650357d684c2"},"cell_type":"markdown","source":"### Data Augmentation\n\nFor Data Augmentation. I will mix train wav, and same length(1 sec) noise(10%) from '_background_noise_'\n"},{"metadata":{"trusted":true,"_uuid":"b59088b2bb5dc1848246dd4320d134ac0c673e65"},"cell_type":"code","source":"#Random pick start point\ndef get_one_noise(noise_num = 0):\n    selected_noise = background_noise[noise_num]\n    start_idx = random.randint(0, len(selected_noise)- 1 - 8000)\n    return selected_noise[start_idx:(start_idx + 8000)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"073bc68512dc809aff793dc03dc380d570631457"},"cell_type":"code","source":"max_ratio = 0.1\nnoised_wav = []\naugment = 1\ndelete_index = []\nfor i in range(augment):\n    new_wav = []\n    noise = get_one_noise(i)\n    for i, s in enumerate(wav_all):\n        if len(s) != 8000:\n            delete_index.append(i)\n            continue\n        s = s + (max_ratio * noise)\n        noised_wav.append(s)\nnp.delete(wav_all, delete_index)\nnp.delete(label_all, delete_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bd94ec47be96cba4be12c60b91ed90044aedea0"},"cell_type":"code","source":"wav_vals = np.array([x for x in wav_all])\nlabel_vals = [x for x in label_all]\nwav_vals.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94ef3d33c62e27cb1fb53a29025c010ec56ffbb4"},"cell_type":"code","source":"labels = copy.deepcopy(label_vals)\nfor _ in range(augment):\n    label_vals = np.concatenate((label_vals, labels), axis = 0)\nlabel_vals = label_vals.reshape(-1,1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e20c12d950b7ff2e8904fc57d5a9d429cd4b130d"},"cell_type":"markdown","source":"Random sampling from unknown wav data\n"},{"metadata":{"trusted":true,"_uuid":"5ee0b02d7cfbcc02ea20d65ee268ba44e01bf2b1"},"cell_type":"code","source":"#knowns audio random sampling\nunknown = unknown_wav\nnp.random.shuffle(unknown_wav)\nunknown = np.array(unknown)\nunknown = unknown[:2000*(augment+1)]\nunknown_label = np.array(['unknown' for _ in range(2000*(augment+1))])\nunknown_label = unknown_label.reshape(2000*(augment+1),1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72a6f2c8663ee74708da42313ed2e8d24c832b3f"},"cell_type":"markdown","source":"May Some wav data has different length. So, Delete it"},{"metadata":{"trusted":true,"_uuid":"70a499bce28fd081b7fa9742eb03851ebe4eb5f1"},"cell_type":"code","source":"delete_index = []\nfor i,w in enumerate(unknown):\n    if len(w) != 8000:\n        delete_index.append(i)\nunknown = np.delete(unknown, delete_index, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a1f8bf0426f373ed063df7f1adbd963943a3fd0"},"cell_type":"markdown","source":"Random sampling from '_background_noise_' \n\nRandom pick background noise \n"},{"metadata":{"trusted":true,"_uuid":"7ef23bb8580b70f2b52e2d91fb183674112d2eba"},"cell_type":"code","source":"#silence audio\nsilence_wav = []\nnum_wav = (2000*(augment+1))//len(background_noise)\nfor i, _ in enumerate(background_noise):\n    for _ in range((2000*(augment+1))//len(background_noise)):\n        silence_wav.append(get_one_noise(i))\nsilence_wav = np.array(silence_wav)\nsilence_label = np.array(['silence' for _ in range(num_wav*len(background_noise))])\nsilence_label = silence_label.reshape(-1,1)\nsilence_wav.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c06ab04ada253e19a0207d280ed3c1a444776cd3"},"cell_type":"code","source":"wav_vals    = np.reshape(wav_vals,    (-1, 8000))\nnoised_wav  = np.reshape(noised_wav,  (-1, 8000))\nunknown       = np.reshape(unknown,   (-1, 8000))\nsilence_wav = np.reshape(silence_wav, (-1, 8000))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e1666c5bf721dbf9e4717be0d469734db9182f2"},"cell_type":"markdown","source":"Check Dimensions"},{"metadata":{"trusted":true,"_uuid":"841b9a9b2a2c38297a7a6d1c93744965755703fa"},"cell_type":"code","source":"print(wav_vals.shape)\nprint(noised_wav.shape)\nprint(unknown.shape)\nprint(silence_wav.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab1aba8c05398669f6e3152d74e00e8a41ee7503"},"cell_type":"code","source":"print(label_vals.shape)\nprint(unknown_label.shape)\nprint(silence_label.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6816376d4450daffeb79114df92d3fb0eafb59a"},"cell_type":"markdown","source":"Concatenate wavs, labels "},{"metadata":{"trusted":true,"_uuid":"7d6afa30a5dda2b9ca4888b36aabe5b89922a121"},"cell_type":"code","source":"wav_vals = np.concatenate((wav_vals, noised_wav), axis = 0)\nwav_vals = np.concatenate((wav_vals, unknown), axis = 0)\nwav_vals = np.concatenate((wav_vals, silence_wav), axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"420f9d709f805349b2635a80dabc383595eac011"},"cell_type":"code","source":"label_vals = np.concatenate((label_vals, unknown_label), axis = 0)\nlabel_vals = np.concatenate((label_vals, silence_label), axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c33c9dc3fe0daa57968e89fa6c22352a8b53f55"},"cell_type":"code","source":"print(len(wav_vals))\nprint(len(label_vals))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63c2da5df2edd06ab1fbdf6c1a50ad02216f0fe5"},"cell_type":"markdown","source":"### Prepare Train"},{"metadata":{"trusted":true,"_uuid":"ba0ab85ffd216b4d91855ea234b3202ba250b8be"},"cell_type":"code","source":"train_wav, test_wav, train_label, test_label = train_test_split(wav_vals, label_vals, \n                                                                    test_size=0.2,\n                                                                    random_state = 1993,\n                                                                   shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e6a9e72fbee7fd3bf612e60d5709b177deacaec"},"cell_type":"code","source":"# Parameters\nlr = 0.001\ngenerations = 20000\nnum_gens_to_wait = 250\nbatch_size = 512\ndrop_out_rate = 0.5\ninput_shape = (8000,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73ffc6cf54f2100c1aa47203f0c33e350198dc3d"},"cell_type":"code","source":"#For Conv1D add Channel\ntrain_wav = train_wav.reshape(-1,8000,1)\ntest_wav = test_wav.reshape(-1,8000,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75fd1a40241009a857b5db9c878c9c665bdc84f2"},"cell_type":"code","source":"label_value = target_list\nlabel_value.append('unknown')\nlabel_value.append('silence')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb801f97812407f60af16a787a0ae01db8e88038"},"cell_type":"code","source":"new_label_value = dict()\nfor i, l in enumerate(label_value):\n    new_label_value[l] = i\nlabel_value = new_label_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65647981d409fb31ac391af3d5d091ef354a0f02"},"cell_type":"code","source":"#Make Label data 'string' -> 'class num'\ntemp = []\nfor v in train_label:\n    temp.append(label_value[v[0]])\ntrain_label = np.array(temp)\n\ntemp = []\nfor v in test_label:\n    temp.append(label_value[v[0]])\ntest_label = np.array(temp)\n\n#Make Label data 'class num' -> 'One hot vector'\ntrain_label = keras.utils.to_categorical(train_label, len(label_value))\ntest_label = keras.utils.to_categorical(test_label, len(label_value))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96b340a033294465394c168e232df012a011ae1a"},"cell_type":"code","source":"print('Train_Wav Demension : ' + str(np.shape(train_wav)))","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"994265f130050666438cd7a14357ef766b279495"},"cell_type":"code","source":"print('Train_Label Demension : ' + str(np.shape(train_label)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edf649f1ec18a201a7a2b2bf9c5115e443fe6b03"},"cell_type":"code","source":"print('Test_Wav Demension : ' + str(np.shape(test_wav)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3bf3f485f3b3f4e44f25ce4ec35861a7b22f866"},"cell_type":"code","source":"print('Test_Label Demension : ' + str(np.shape(test_label)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"557d3f766acfe47197813f0301d0b562cf3388be"},"cell_type":"code","source":"print('Number Of Labels : ' + str(len(label_value)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3ac90355cbf04993a8b090483164d0e86401855"},"cell_type":"code","source":"#Conv1D Model\ninput_tensor = Input(shape=(input_shape))\n\nx = layers.Conv1D(8, 11, padding='valid', activation='relu', strides=1)(input_tensor)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(16, 7, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(32, 5, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(64, 5, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Conv1D(128, 3, padding='valid', activation='relu', strides=1)(x)\nx = layers.MaxPooling1D(2)(x)\nx = layers.Flatten()(x)\nx = layers.Dense(256, activation='relu')(x)\nx = layers.Dropout(drop_out_rate)(x)\nx = layers.Dense(128, activation='relu')(x)\nx = layers.Dropout(drop_out_rate)(x)\noutput_tensor = layers.Dense(len(label_value), activation='softmax')(x)\n\nmodel = tf.keras.Model(input_tensor, output_tensor)\n\nmodel.compile(loss=keras.losses.categorical_crossentropy,\n             optimizer=keras.optimizers.Adam(lr = lr),\n             metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c1c50793d9d2c12af1cbf1029daca221afdaff9"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"45ce60c3949727441c5c75dabb3575cb95aa59be"},"cell_type":"markdown","source":"### Train!"},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"9b61710f92afdf22a09a07b9dd9d3ddc3e75c05b"},"cell_type":"code","source":"history = model.fit(train_wav, train_label, validation_data=[test_wav, test_label],\n          batch_size=batch_size, \n          epochs=100,\n          verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cbb0f06ea603cb0e5ef7b9893080ecfdd082864"},"cell_type":"code","source":"plt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}