{"cells":[{"source":"# importing all the dependencies\nimport pandas as pd # data frame\nimport numpy as np # matrix math\nfrom glob import glob # file handling\nimport librosa # audio manipulation\nfrom sklearn.utils import shuffle # shuffling of data\nimport os # interation with the OS\nfrom random import sample # random selection\nfrom tqdm import tqdm","outputs":[],"cell_type":"code","metadata":{"_uuid":"ad0c04a55c8d9e9df6f5f8c0e0679dd6cad7f43e","_cell_guid":"d68bbe1e-d1c2-4570-aa3d-127ccc432409","collapsed":true},"execution_count":1},{"source":"# fixed param\nPATH = '../input/train/audio/'","outputs":[],"cell_type":"code","metadata":{"_uuid":"bdd3a75d7c58581308e5576fe4e0f32e640f9d64","_cell_guid":"8ec0273e-3f66-4ae5-b1f1-a8ec5714cfa6","collapsed":true},"execution_count":3},{"source":"def load_files(path):\n\t# write the complete file loading function here, this will return\n\t# a dataframe having files and labels\n\t# loading the files\n\ttrain_labels = os.listdir(PATH)\n\ttrain_labels.remove('_background_noise_')\n\n\tlabels_to_keep = ['yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go', 'silence']\n\n\ttrain_file_labels = dict()\n\tfor label in train_labels:\n\t\tfiles = os.listdir(PATH + '/' + label)\n\t\tfor f in files:\n\t\t\ttrain_file_labels[label + '/' + f] = label\n\n\ttrain = pd.DataFrame.from_dict(train_file_labels, orient='index')\n\ttrain = train.reset_index(drop=False)\n\ttrain = train.rename(columns={'index': 'file', 0: 'folder'})\n\ttrain = train[['folder', 'file']]\n\ttrain = train.sort_values('file')\n\ttrain = train.reset_index(drop=True)\n\n\tdef remove_label_from_file(label, fname):\n\t\treturn path + label + '/' + fname[len(label)+1:]\n\n\ttrain['file'] = train.apply(lambda x: remove_label_from_file(*x), axis=1)\n\ttrain['label'] = train['folder'].apply(lambda x: x if x in labels_to_keep else 'unknown')\n\n\tlabels_to_keep.append('unknown')\n\n\treturn train, labels_to_keep","outputs":[],"cell_type":"code","metadata":{"_uuid":"9aeb343d33a929aacfc9fa8a5739a020b10ca23c","_cell_guid":"d3e59f74-0617-4216-8854-91f5751eb07c","collapsed":true},"execution_count":4},{"source":"train, labels_to_keep = load_files(PATH)\n\n# making word2id dict\nword2id = dict((c,i) for i,c in enumerate(sorted(labels_to_keep)))\n\n# get some files which will be labeled as unknown\nunk_files = train.loc[train['label'] == 'unknown']['file'].values\nunk_files = sample(list(unk_files), 1000)","outputs":[],"cell_type":"code","metadata":{"_uuid":"54636d29d74e50e339de4db5b884ef54bc6f8642","_cell_guid":"8a3fd5dc-5742-454b-8c87-b3e4ab590e28","collapsed":true},"execution_count":16},{"source":"word2id","outputs":[],"cell_type":"code","metadata":{},"execution_count":17},{"source":"unk_files[:10]","outputs":[],"cell_type":"code","metadata":{"_uuid":"9223b049b6dce69b3d27e2d4c066e9995ac7a05d","_cell_guid":"9397fcfc-6421-4ae2-9497-6afffbb7f837"},"execution_count":6},{"source":"train.sample(5)","outputs":[],"cell_type":"code","metadata":{"_uuid":"f93623173a0a9db436dda590e9e2f28c1a8c6fb4","_cell_guid":"0b260224-7659-4812-89f2-6712d9cbd1a4"},"execution_count":7},{"source":"# Writing functions to extract the data, script from kdnuggets: \n# www.kdnuggets.com/2016/09/urban-sound-classification-neural-networks-tensorflow.html\ndef extract_feature(path):\n\tX, sample_rate = librosa.load(path)\n\tstft = np.abs(librosa.stft(X))\n\tmfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=40).T,axis=0)\n\tchroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T,axis=0)\n\tmel = np.mean(librosa.feature.melspectrogram(X, sr=sample_rate).T,axis=0)\n\tcontrast = np.mean(librosa.feature.spectral_contrast(S=stft, sr=sample_rate).T,axis=0)\n\ttonnetz = np.mean(librosa.feature.tonnetz(y=librosa.effects.harmonic(X), sr=sample_rate).T,axis=0)\n\treturn mfccs,chroma,mel,contrast,tonnetz\n\ndef parse_audio_files(files, word2id, unk = False):\n    # n: number of classes\n    features = np.empty((0,193))\n    one_hot = np.zeros(shape = (len(files), word2id[max(word2id)]))\n    print(one_hot.shape)\n    for i in tqdm(range(len(files))):\n        f = files[i]\n        mfccs, chroma, mel, contrast,tonnetz = extract_feature(f)\n        ext_features = np.hstack([mfccs,chroma,mel,contrast,tonnetz])\n        features = np.vstack([features,ext_features])\n        if unk == True:\n            l = word2id['unknown']\n            one_hot[i][l] = 1.\n        else:\n            l = word2id[f.split('/')[-2]]\n            one_hot[i][l] = 1.\n    return np.array(features), one_hot","outputs":[],"cell_type":"code","metadata":{"_uuid":"8c188669f712d2fb23256c1e602845e555655d34","_cell_guid":"c2714134-3ef6-4f0a-8802-0c535a0bc7b6","collapsed":true},"execution_count":13},{"source":"files = train.loc[train['label'] != 'unknown']['file'].values\nprint(len(files))\nprint(files[:10])","outputs":[],"cell_type":"code","metadata":{},"execution_count":10},{"source":"## Playing around with the single audio clip\nWe now look at a single audio clip and see how it goes.","cell_type":"markdown","metadata":{"_uuid":"efadf91e5f75beb0dfc0e70f4710d3a4247654b1","_cell_guid":"6fdd84ad-ecf8-4caf-bf2e-4f44806855cc"}},{"source":"# playing around with the data for now\ntrain_audio_path = '../input/train/audio/'\nfilename = '/tree/24ed94ab_nohash_0.wav' # --> 'Yes'\nsample_rate, audio = wavfile.read(str(train_audio_path) + filename)","outputs":[],"cell_type":"code","metadata":{"_uuid":"37a05080d1dad896fa908ab62bcf7bdedc0481b0","_cell_guid":"0f0f4014-28f3-4f9b-8c78-de05de697376","collapsed":true},"execution_count":null},{"source":"plt.figure(figsize = (15, 4))\nplt.plot(audio)\nipd.Audio(audio, rate=sample_rate)","outputs":[],"cell_type":"code","metadata":{"_uuid":"820b899ac73d5c7b3da57c620a822ef28dc0d08d","_cell_guid":"07f96a24-9b35-426d-908d-8a45de3813da","collapsed":true},"execution_count":null},{"source":"# goto: https://medium.com/@ageitgey/machine-learning-is-fun-part-6-how-to-do-speech-recognition-with-deep-learning-28293c162f7a\n# We convert it into chunks of 20ms each i.e. units of 320 \naudio_chunks = []\nn_chunks = int(audio.shape[0]/320)\nfor i in range(n_chunks):\n    chunk = audio[i*320: (i+1)*320]\n    audio_chunks.append(chunk)\naudio_chunk = np.array(audio_chunks)","outputs":[],"cell_type":"code","metadata":{"_uuid":"08c9d6342fb369d1542935dc8273a9a9f86b8d8c","_cell_guid":"a57800b7-7b66-4876-ae16-1e552e5f75e9","collapsed":true},"execution_count":null},{"source":"# we now convert it to spertogram\n# goto: https://www.kaggle.com/davids1992/data-visualization-and-investigation\ndef log_specgram(audio, sample_rate, window_size=10,\n                 step_size=10, eps=1e-10):\n    nperseg = int(round(window_size * sample_rate / 1e3))\n    noverlap = int(round(step_size * sample_rate / 1e3))\n    _, _, spec = signal.spectrogram(audio,\n                                    fs=sample_rate,\n                                    window='hann',\n                                    nperseg=nperseg,\n                                    noverlap=noverlap,\n                                    detrend=False)\n    return np.log(spec.T.astype(np.float32) + eps)","outputs":[],"cell_type":"code","metadata":{"_uuid":"f5e1bdb8c1ff3dec04036d32deaa43f3a09007e7","_cell_guid":"073c82e3-f7bf-445e-bde7-2f02c6bba686","collapsed":true},"execution_count":null},{"source":"spectrogram = log_specgram(audio, sample_rate, 10, 0)\nspec = spectrogram.T\nprint(spec.shape)\nplt.figure(figsize = (15,4))\nplt.imshow(spec, aspect='auto', origin='lower')","outputs":[],"cell_type":"code","metadata":{"_uuid":"3b1f903bc0172c4274916879ace58e77933b2fd6","_cell_guid":"599e9f56-7df5-4e3a-986f-e4e0b9f80335","collapsed":true},"execution_count":null},{"source":"## Making the data\nNow that we know about the shape of the data, we will finally make the total processed data.","cell_type":"markdown","metadata":{"_uuid":"f1f9cfa625e58f0c28702a4570a63771680ff7e5","_cell_guid":"77f92371-d7dd-4d27-bc2e-c95a5d65ec8b","collapsed":true}},{"source":"# make labels and convert them into one hot encodings\nlabels = sorted(labels_to_keep)\nword2id = dict((c,i) for i,c in enumerate(labels))\nlabel = train['label'].values\nlabel = [word2id[l] for l in label]\nprint(labels)\ndef make_one_hot(seq, n):\n    # n --> vocab size\n    seq_new = np.zeros(shape = (len(seq), n))\n    for i,s in enumerate(seq):\n        seq_new[i][s] = 1.\n    return seq_new\none_hot_l = make_one_hot(label, 12)","outputs":[],"cell_type":"code","metadata":{"_uuid":"452337cac89c2600175e6e5846d794e52a8c8ec5","_cell_guid":"c8f978dc-e5d5-4139-ad94-b44f7b4e3de6","collapsed":true},"execution_count":null},{"source":"print(one_hot_l[10:15])","outputs":[],"cell_type":"code","metadata":{"_uuid":"41e7c2ed98097743048f1521ac7ef897cd751a18","_cell_guid":"7d255b96-f111-4b94-9a19-70edf6d537d0","collapsed":true},"execution_count":null},{"source":"one_hot_l[0]","outputs":[],"cell_type":"code","metadata":{"_uuid":"b31bb2ce6010b9992ec3103d769459352fcf51ba","_cell_guid":"e426d81d-d85b-4f2e-aab8-28a12482a83d","collapsed":true},"execution_count":null},{"source":"# getting all the paths to the files\npaths = []\nfolders = train['folder']\nfiles = train['file']\nfor i in range(len(files)):\n    path = '../input/train/audio/' + str(folders[i]) + '/' + str(files[i])\n    paths.append(path)","outputs":[],"cell_type":"code","metadata":{"_uuid":"04dc4d6d286d43b9d2e2a4db71ae2dd980a69124","_cell_guid":"83c90011-3b4d-434e-a10e-c67c02c8d712","collapsed":true},"execution_count":null},{"source":"def audio_to_data(path):\n    # we take a single path and convert it into data\n    sample_rate, audio = wavfile.read(path)\n    spectrogram = log_specgram(audio, sample_rate, 10, 0)\n    return spectrogram.T\n\ndef paths_to_data(paths,labels):\n    data = np.zeros(shape = (len(paths), 81, 100))\n    indexes = []\n    for i in tqdm(range(len(paths))):\n        audio = audio_to_data(paths[i])\n        if audio.shape != (81,100):\n            indexes.append(i)\n        else:\n            data[i] = audio\n    final_labels = [l for i,l in enumerate(labels) if i not in indexes]\n    print('Number of instances with inconsistent shape:', len(indexes))\n    return data[:len(data)-len(indexes)], final_labels, indexes","outputs":[],"cell_type":"code","metadata":{"_uuid":"7e50caf5ed399c323c20b142bda1b7e48eb95f66","_cell_guid":"9fa309a3-f0e6-42b1-ac9e-3f980328fdbc","collapsed":true},"execution_count":null},{"source":"d,l,indexes = paths_to_data(paths,one_hot_l)","outputs":[],"cell_type":"code","metadata":{"_uuid":"087b898b3c9d1bc85b1659df9b5a2482604dc35b","_cell_guid":"d1553511-3949-429b-8019-30c41633937e","collapsed":true},"execution_count":null},{"source":"labels = np.zeros(shape = [d.shape[0], len(l[0])])\nfor i,array in enumerate(l):\n    for j, element in enumerate(array):\n        labels[i][j] = element\nprint(labels.shape)","outputs":[],"cell_type":"code","metadata":{"_uuid":"4f0529718592b8647cf5b0ad805f9454a67e68e9","_cell_guid":"c9044104-3725-49f7-a932-730d291d56e8","collapsed":true},"execution_count":null},{"source":"print(d.shape)\nprint(labels.shape)","outputs":[],"cell_type":"code","metadata":{"_uuid":"4af5fd8d60521c50296b9c0aebbf7806e7885339","_cell_guid":"9fd471cc-d645-487d-92ac-eaad221e06c6","collapsed":true},"execution_count":null},{"source":"d,labels = shuffle(d,labels)","outputs":[],"cell_type":"code","metadata":{"_uuid":"13527981e2544cebfb8db0698056963b257f780a","_cell_guid":"7db7002a-fa5e-4074-b4fb-354f41a3a5df","collapsed":true},"execution_count":null},{"source":"print(d[0].shape)\nprint(labels[0].shape)","outputs":[],"cell_type":"code","metadata":{"_uuid":"66c5c03b294fdfcd11041e8c9eaae8d7ad4d27f1","_cell_guid":"d4c84402-7a3e-40ba-9855-2c9940e9af84","collapsed":true},"execution_count":null},{"source":"## Machine learning model\nUsing a LSTM network to determine the text","cell_type":"markdown","metadata":{"_uuid":"a1eed9a6f5646e0935457e36df5be9f627634e82","_cell_guid":"f15a16a3-7cdd-45e0-b61e-cf81ca7d6939"}},{"source":"from keras.models import Sequential\nfrom keras.layers import LSTM, Dense, Dropout","outputs":[],"cell_type":"code","metadata":{"_uuid":"ea41ed6543048d78b78885729314295ddd4f281b","_cell_guid":"261767dd-0aa0-4216-bc31-19c4fea6bf3c","collapsed":true},"execution_count":null},{"source":"model = Sequential()\nmodel.add(LSTM(256, input_shape = (81, 100)))\n# model.add(Dense(1028))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(128))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(12, activation = 'softmax'))\nmodel.compile(optimizer = 'Adam', loss = 'mean_squared_error', metrics = ['accuracy'])","outputs":[],"cell_type":"code","metadata":{"_uuid":"b0f6df7151bf99a12107c81c6ed93413b31d493d","_cell_guid":"a1d3e80e-5baa-4138-829e-59df333c5af1","collapsed":true},"execution_count":null},{"source":"model.summary()","outputs":[],"cell_type":"code","metadata":{"_uuid":"37f28487072c647dee1244486e911b83bef62a19","_cell_guid":"14aae093-ee2c-4327-be45-5f9a967fbf83","collapsed":true},"execution_count":null},{"source":"model.fit(d, labels, batch_size = 1024, epochs = 10)","outputs":[],"cell_type":"code","metadata":{"_uuid":"3105b699495b6575f181416a57a2ee57bcda7d36","_cell_guid":"368722ad-3634-4b42-906c-91e4a99fedbc","collapsed":true},"execution_count":null},{"source":"## Add further for testing modules\nAdd modules for testing and saving the files, will keeo improving the model in the future.","cell_type":"markdown","metadata":{"_uuid":"ab37532a5628a1f70314c263f07dbf935d208999","_cell_guid":"d5932f12-b408-4e30-b204-fc139ae5e94e","collapsed":true}},{"source":"","outputs":[],"cell_type":"code","metadata":{"_uuid":"105317c5c076a435f6be8a3b34bf75bf347078d1","_cell_guid":"67f9b41e-72b2-4dd6-b511-52462f9b9a66","collapsed":true},"execution_count":null}],"nbformat_minor":1,"nbformat":4,"metadata":{"language_info":{"file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","name":"python","nbconvert_exporter":"python","version":"3.6.3","mimetype":"text/x-python"},"kernelspec":{"name":"python3","language":"python","display_name":"Python 3"}}}