{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import the libraries\n","metadata":{"execution":{"iopub.status.busy":"2022-02-11T15:15:42.09471Z","iopub.execute_input":"2022-02-11T15:15:42.095103Z","iopub.status.idle":"2022-02-11T15:15:42.099724Z","shell.execute_reply.started":"2022-02-11T15:15:42.095016Z","shell.execute_reply":"2022-02-11T15:15:42.098645Z"}}},{"cell_type":"code","source":"!pip install pyunpack\n!pip install patool\n!pip install py7zr\n!pip install sounddevice\n!pip install noisereduce\n!pip install librosa\n! pip install python_speech_features\n! pip install tensorflow==2.4\n! pip install malaya_speech\n! pip install webrtcvad","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:46:13.482252Z","iopub.execute_input":"2022-02-27T18:46:13.482531Z","iopub.status.idle":"2022-02-27T18:48:06.681103Z","shell.execute_reply.started":"2022-02-27T18:46:13.482467Z","shell.execute_reply":"2022-02-27T18:48:06.678482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom py7zr import unpack_7zarchive\nimport shutil\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport librosa\nimport IPython.display as ipd\nfrom scipy.io import wavfile\n\nimport noisereduce as nr\nimport tensorflow \nfrom malaya_speech import Pipeline\n\nimport malaya_speech\nimport os\n\nfrom python_speech_features import mfcc\n\nfrom sklearn.preprocessing import LabelEncoder\nimport seaborn as sn","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:48:06.698078Z","iopub.execute_input":"2022-02-27T18:48:06.700273Z","iopub.status.idle":"2022-02-27T18:48:11.516424Z","shell.execute_reply.started":"2022-02-27T18:48:06.700227Z","shell.execute_reply":"2022-02-27T18:48:11.515550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)\nshutil.unpack_archive('/kaggle/input/tensorflow-speech-recognition-challenge/train.7z', '/kaggle/working/tensorflow-speech-recognition-challenge/train/')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:48:11.518258Z","iopub.execute_input":"2022-02-27T18:48:11.518612Z","iopub.status.idle":"2022-02-27T18:50:40.987752Z","shell.execute_reply.started":"2022-02-27T18:48:11.518560Z","shell.execute_reply":"2022-02-27T18:50:40.986709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from pyunpack import Archive\n# import shutil\n# if not os.path.exists('/kaggle/working/tensorflow-speech-recognition-challenge/train/'):\n#     os.makedirs('/kaggle/working/tensorflow-speech-recognition-challenge/train/')\n# Archive('/kaggle/input/tensorflow-speech-recognition-challenge/train.7z').extractall('/kaggle/working/tensorflow-speech-recognition-challenge/train/')\n\n#for dirname, _, filenames in os.walk('/kaggle/working/tensorflow-speech-recognition-challenge/train/train/audio'):\n #   for filename in filename[:5]:\n  #      print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:40.989553Z","iopub.execute_input":"2022-02-27T18:50:40.990030Z","iopub.status.idle":"2022-02-27T18:50:40.994878Z","shell.execute_reply.started":"2022-02-27T18:50:40.989992Z","shell.execute_reply":"2022-02-27T18:50:40.993346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <center> Implementing the Speech Recognition Model in Python\n**Dataset used for our Speech Recognition Project**\n\n__You can download the dataset from__ [here](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge).\n    \nTensorFlow recently released the Speech Commands Datasets. It includes 65,000 one-second long utterances of 30 short words, by thousands of different people. We’ll build a speech recognition system that understands simple spoken commands. <br>    ","metadata":{}},{"cell_type":"markdown","source":"**Data Exploration and Visualization**\n\nData Exploration and Visualization helps us to understand the data as well as pre-processing steps in a better way. \n\n**Visualization of Audio signal in time series domain**\n\nNow, we’ll visualize the audio signal in the time series domain:","metadata":{}},{"cell_type":"code","source":"train_audio_path = '/kaggle/working/tensorflow-speech-recognition-challenge/train/train/audio/'","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:40.996332Z","iopub.execute_input":"2022-02-27T18:50:40.996676Z","iopub.status.idle":"2022-02-27T18:50:41.005466Z","shell.execute_reply.started":"2022-02-27T18:50:40.996640Z","shell.execute_reply":"2022-02-27T18:50:41.004529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Accessing each file in data**","metadata":{}},{"cell_type":"code","source":"#!apt-get install -y p7zip-full\n#!7z x ../input/tensorflow-speech-recognition-challenge/train.7z","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:41.010581Z","iopub.execute_input":"2022-02-27T18:50:41.011240Z","iopub.status.idle":"2022-02-27T18:50:41.016906Z","shell.execute_reply.started":"2022-02-27T18:50:41.011202Z","shell.execute_reply":"2022-02-27T18:50:41.016050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples, sample_rate = librosa.load(train_audio_path+'on/5a3712c9_nohash_1.wav', sr = 16000)\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + '../input/train/audio/on/0a7c2a8d_nohash_0.wav')\nax1.set_xlabel('time')\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:41.019622Z","iopub.execute_input":"2022-02-27T18:50:41.020773Z","iopub.status.idle":"2022-02-27T18:50:41.287449Z","shell.execute_reply.started":"2022-02-27T18:50:41.020731Z","shell.execute_reply":"2022-02-27T18:50:41.286556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sampling rate **\n\nLet us now look at the sampling rate of the audio signals","metadata":{}},{"cell_type":"code","source":"ipd.Audio(samples, rate=sample_rate)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:41.290092Z","iopub.execute_input":"2022-02-27T18:50:41.290587Z","iopub.status.idle":"2022-02-27T18:50:41.298499Z","shell.execute_reply.started":"2022-02-27T18:50:41.290547Z","shell.execute_reply":"2022-02-27T18:50:41.297487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_rate)\nsig1=samples\nfs=sample_rate\nsr=fs","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:41.300622Z","iopub.execute_input":"2022-02-27T18:50:41.301186Z","iopub.status.idle":"2022-02-27T18:50:41.306556Z","shell.execute_reply.started":"2022-02-27T18:50:41.301141Z","shell.execute_reply":"2022-02-27T18:50:41.305592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time = np.linspace(0, len(sig1 - 1) / fs, len(sig1 - 1))\nreduced_noise1 = nr.reduce_noise(y=sig1, sr=fs,stationary=True)\nplt.plot(time, reduced_noise1)  # plot in seconds\n#reduced_noise2 = nr.reduce_noise(y=sig2, sr=fs,stationary=True)\n#plt.plot(time, reduced_noise2)  # plot in seconds\n#plt.title(\"Voice Signal\")\nplt.xlabel(\"Time [seconds]\")\nplt.ylabel(\"Voice amplitude\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:41.308143Z","iopub.execute_input":"2022-02-27T18:50:41.308720Z","iopub.status.idle":"2022-02-27T18:50:42.013373Z","shell.execute_reply.started":"2022-02-27T18:50:41.308682Z","shell.execute_reply":"2022-02-27T18:50:42.012590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(reduced_noise1, rate=sample_rate)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.014634Z","iopub.execute_input":"2022-02-27T18:50:42.014975Z","iopub.status.idle":"2022-02-27T18:50:42.022345Z","shell.execute_reply.started":"2022-02-27T18:50:42.014940Z","shell.execute_reply":"2022-02-27T18:50:42.021528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Silence Removal\nvad = malaya_speech.vad.webrtc()\ny=reduced_noise1\ny_= malaya_speech.resample(y, sr, 16000)\ny_ = malaya_speech.astype.float_to_int(y_)\nframes = malaya_speech.generator.frames(y, 30, sr)\nframes_ = list(malaya_speech.generator.frames(y_, 30, 16000, append_ending_trail = False))\nframes_webrtc = [(frames[no], vad(frame)) for no, frame in enumerate(frames_)]\ny_ = malaya_speech.combine.without_silent(frames_webrtc)\ny_","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.023662Z","iopub.execute_input":"2022-02-27T18:50:42.024248Z","iopub.status.idle":"2022-02-27T18:50:42.228917Z","shell.execute_reply.started":"2022-02-27T18:50:42.024210Z","shell.execute_reply":"2022-02-27T18:50:42.227926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(y_, rate = sr )","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.230085Z","iopub.execute_input":"2022-02-27T18:50:42.230398Z","iopub.status.idle":"2022-02-27T18:50:42.240501Z","shell.execute_reply.started":"2022-02-27T18:50:42.230369Z","shell.execute_reply":"2022-02-27T18:50:42.238770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero = np.zeros((1*sr-y_.shape[0]))\nsignal = np.concatenate((y_,zero))\nsignal.shape\ntime = np.linspace(0, len(signal - 1) / fs, len(signal - 1))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.242635Z","iopub.execute_input":"2022-02-27T18:50:42.243255Z","iopub.status.idle":"2022-02-27T18:50:42.249532Z","shell.execute_reply.started":"2022-02-27T18:50:42.243214Z","shell.execute_reply":"2022-02-27T18:50:42.248637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(time,signal)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.251092Z","iopub.execute_input":"2022-02-27T18:50:42.251641Z","iopub.status.idle":"2022-02-27T18:50:42.397363Z","shell.execute_reply.started":"2022-02-27T18:50:42.251597Z","shell.execute_reply":"2022-02-27T18:50:42.396567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=os.listdir(train_audio_path)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.398649Z","iopub.execute_input":"2022-02-27T18:50:42.399153Z","iopub.status.idle":"2022-02-27T18:50:42.404454Z","shell.execute_reply.started":"2022-02-27T18:50:42.399113Z","shell.execute_reply":"2022-02-27T18:50:42.403550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#find count of each label and plot bar graph\nno_of_recordings=[]\nfor label in labels:\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label) if f.endswith('.wav')]\n    no_of_recordings.append(len(waves))\n    \n#plot\nplt.figure(figsize=(30,5))\nindex = np.arange(len(labels))\nplt.bar(index, no_of_recordings)\nplt.xlabel('Commands', fontsize=12)\nplt.ylabel('No of recordings', fontsize=12)\nplt.xticks(index, labels, fontsize=15, rotation=60)\nplt.title('No. of recordings for each command')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.405893Z","iopub.execute_input":"2022-02-27T18:50:42.406373Z","iopub.status.idle":"2022-02-27T18:50:42.742528Z","shell.execute_reply.started":"2022-02-27T18:50:42.406338Z","shell.execute_reply":"2022-02-27T18:50:42.741722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Words used","metadata":{}},{"cell_type":"code","source":"labels=[\"on\",\"off\",\"up\",\"down\",\"go\",\"stop\",\"left\",\"right\"]","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.743757Z","iopub.execute_input":"2022-02-27T18:50:42.744113Z","iopub.status.idle":"2022-02-27T18:50:42.748436Z","shell.execute_reply.started":"2022-02-27T18:50:42.744076Z","shell.execute_reply":"2022-02-27T18:50:42.747343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise_audio_path = '../input/hospital-ambient-noise/Hospital noise original/Hospital noise original/'\nnoise_path='../input/pinknoise/'","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.749634Z","iopub.execute_input":"2022-02-27T18:50:42.750041Z","iopub.status.idle":"2022-02-27T18:50:42.758041Z","shell.execute_reply.started":"2022-02-27T18:50:42.750005Z","shell.execute_reply":"2022-02-27T18:50:42.757232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_of_other=['can not be identified']","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.759428Z","iopub.execute_input":"2022-02-27T18:50:42.759791Z","iopub.status.idle":"2022-02-27T18:50:42.767144Z","shell.execute_reply.started":"2022-02-27T18:50:42.759754Z","shell.execute_reply":"2022-02-27T18:50:42.766238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Preprocessing the audio waves**\n\nlet us read the audio waves and use the below-preprocessing steps :\n\n* Noise Reduction\n* Silence Removal\n* Extracting MFCCs\n\nLet us define these preprocessing steps in the below code snippet:","metadata":{}},{"cell_type":"code","source":"sr=16000\nvad = malaya_speech.vad.webrtc()\nall_wave = []\nall_label = []\nfor label in labels:\n    print(label)\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label) if f.endswith('.wav')]\n    for wav in waves:\n        samples, sample_rate = librosa.load(train_audio_path + '/' + label + '/' + wav, sr = 16000)\n        samples = nr.reduce_noise(y=samples, sr=sr,stationary=True)\n        y_= malaya_speech.resample(samples, sr, 16000)\n        y_ = malaya_speech.astype.float_to_int(y_)\n        frames = malaya_speech.generator.frames(samples, 30, sr)\n        frames_ = list(malaya_speech.generator.frames(y_, 30, 16000, append_ending_trail = False))\n        frames_webrtc = [(frames[no], vad(frame)) for no, frame in enumerate(frames_)]\n        y_ = malaya_speech.combine.without_silent(frames_webrtc)\n        zero = np.zeros(((1*sr+4000)-y_.shape[0]))\n        signal = np.concatenate((y_,zero))\n        all_wave.append(signal)\n        all_label.append(label)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:50:42.768371Z","iopub.execute_input":"2022-02-27T18:50:42.768736Z","iopub.status.idle":"2022-02-27T18:59:12.526628Z","shell.execute_reply.started":"2022-02-27T18:50:42.768710Z","shell.execute_reply":"2022-02-27T18:59:12.525810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.array(all_wave).shape)\nprint(np.array(all_label).shape)\nfs=16000\nsr=fs\ntime = np.linspace(0, len(signal - 1) / fs, len(signal - 1))\nplt.plot(time,np.array(all_wave)[2000,:])\nprint(np.array(all_label)[2000])\nipd.Audio(np.array(all_wave)[2000,:], rate = sr )","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:59:12.527977Z","iopub.execute_input":"2022-02-27T18:59:12.528328Z","iopub.status.idle":"2022-02-27T18:59:21.300455Z","shell.execute_reply.started":"2022-02-27T18:59:12.528293Z","shell.execute_reply":"2022-02-27T18:59:21.299683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_mfcc=[]\nfor wave in all_wave:\n    i=0\n    mfcc_feat = mfcc(wave , fs, winlen=256/fs, winstep=256/(2*fs), numcep=13, nfilt=26, nfft=256,\n                 lowfreq=0, highfreq=fs/2, preemph=0.97, ceplifter=22, appendEnergy=True, winfunc=np.hamming)\n    mfcc_feat= np.transpose(mfcc_feat)\n    all_mfcc.append(mfcc_feat)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:59:21.305075Z","iopub.execute_input":"2022-02-27T18:59:21.305330Z","iopub.status.idle":"2022-02-27T19:00:17.607919Z","shell.execute_reply.started":"2022-02-27T18:59:21.305305Z","shell.execute_reply":"2022-02-27T19:00:17.606892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.array(all_mfcc).shape)\nprint(np.array(all_label).shape)\nd1=np.array(all_mfcc).shape[1]\nd2=np.array(all_mfcc).shape[2]\nd=d1*d2\nprint(d)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:17.610636Z","iopub.execute_input":"2022-02-27T19:00:17.611255Z","iopub.status.idle":"2022-02-27T19:00:18.026441Z","shell.execute_reply.started":"2022-02-27T19:00:17.611214Z","shell.execute_reply":"2022-02-27T19:00:18.025556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#op_mfcc=np.array(all_mfcc)\n#op_mfcc=op_mfcc.reshape(np.array(all_mfcc).shape[0],-1)\n#op_mfcc.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.027740Z","iopub.execute_input":"2022-02-27T19:00:18.028087Z","iopub.status.idle":"2022-02-27T19:00:18.032360Z","shell.execute_reply.started":"2022-02-27T19:00:18.028041Z","shell.execute_reply":"2022-02-27T19:00:18.031287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir(noise_path))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.033723Z","iopub.execute_input":"2022-02-27T19:00:18.034283Z","iopub.status.idle":"2022-02-27T19:00:18.096585Z","shell.execute_reply.started":"2022-02-27T19:00:18.034244Z","shell.execute_reply":"2022-02-27T19:00:18.095806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir(noise_audio_path))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.097902Z","iopub.execute_input":"2022-02-27T19:00:18.098265Z","iopub.status.idle":"2022-02-27T19:00:18.157730Z","shell.execute_reply.started":"2022-02-27T19:00:18.098226Z","shell.execute_reply":"2022-02-27T19:00:18.156961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise_waves_1 = os.listdir(noise_path)\nnoise_waves_2 =os.listdir(noise_audio_path)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.160371Z","iopub.execute_input":"2022-02-27T19:00:18.160633Z","iopub.status.idle":"2022-02-27T19:00:18.167173Z","shell.execute_reply.started":"2022-02-27T19:00:18.160608Z","shell.execute_reply":"2022-02-27T19:00:18.166270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise_files1=np.array(noise_waves_1)\nnoise_files2=np.array(noise_waves_2)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.168774Z","iopub.execute_input":"2022-02-27T19:00:18.169228Z","iopub.status.idle":"2022-02-27T19:00:18.175152Z","shell.execute_reply.started":"2022-02-27T19:00:18.169190Z","shell.execute_reply":"2022-02-27T19:00:18.174101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise_files=np.hstack((noise_files1, noise_files2))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.176712Z","iopub.execute_input":"2022-02-27T19:00:18.177183Z","iopub.status.idle":"2022-02-27T19:00:18.183540Z","shell.execute_reply.started":"2022-02-27T19:00:18.177148Z","shell.execute_reply":"2022-02-27T19:00:18.182743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise_files[0]","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.184964Z","iopub.execute_input":"2022-02-27T19:00:18.185378Z","iopub.status.idle":"2022-02-27T19:00:18.194634Z","shell.execute_reply.started":"2022-02-27T19:00:18.185339Z","shell.execute_reply":"2022-02-27T19:00:18.193539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples, sample_rate = librosa.load(noise_audio_path+noise_files[1000], sr = 16000)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:18.196012Z","iopub.execute_input":"2022-02-27T19:00:18.196425Z","iopub.status.idle":"2022-02-27T19:00:19.009828Z","shell.execute_reply.started":"2022-02-27T19:00:18.196371Z","shell.execute_reply":"2022-02-27T19:00:19.008891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(samples, rate = 16000 )","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:19.011187Z","iopub.execute_input":"2022-02-27T19:00:19.011525Z","iopub.status.idle":"2022-02-27T19:00:19.026743Z","shell.execute_reply.started":"2022-02-27T19:00:19.011489Z","shell.execute_reply":"2022-02-27T19:00:19.025946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_rate)\nsig1=samples\nfs=sample_rate\nsr=fs","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:19.027922Z","iopub.execute_input":"2022-02-27T19:00:19.028257Z","iopub.status.idle":"2022-02-27T19:00:19.037055Z","shell.execute_reply.started":"2022-02-27T19:00:19.028225Z","shell.execute_reply":"2022-02-27T19:00:19.036020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Silence Removal\nvad = malaya_speech.vad.webrtc()\ny=samples\ny_= malaya_speech.resample(y, sr, 16000)\ny_ = malaya_speech.astype.float_to_int(y_)\nframes = malaya_speech.generator.frames(y, 30, sr)\nframes_ = list(malaya_speech.generator.frames(y_, 30, 16000, append_ending_trail = False))\nframes_webrtc = [(frames[no], vad(frame)) for no, frame in enumerate(frames_)]\ny_ = malaya_speech.combine.without_silent(frames_webrtc)\ny_=y_[:16000]","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:19.038698Z","iopub.execute_input":"2022-02-27T19:00:19.039265Z","iopub.status.idle":"2022-02-27T19:00:19.084737Z","shell.execute_reply.started":"2022-02-27T19:00:19.039226Z","shell.execute_reply":"2022-02-27T19:00:19.083901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero = np.zeros((1*sr-y_.shape[0]))\nsignal = np.concatenate((y_,zero))\nsignal.shape\ntime = np.linspace(0, len(signal - 1) / fs, len(signal - 1))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:19.085985Z","iopub.execute_input":"2022-02-27T19:00:19.086363Z","iopub.status.idle":"2022-02-27T19:00:19.094339Z","shell.execute_reply.started":"2022-02-27T19:00:19.086321Z","shell.execute_reply":"2022-02-27T19:00:19.093510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(time,signal)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:19.096025Z","iopub.execute_input":"2022-02-27T19:00:19.096496Z","iopub.status.idle":"2022-02-27T19:00:19.242210Z","shell.execute_reply.started":"2022-02-27T19:00:19.096455Z","shell.execute_reply":"2022-02-27T19:00:19.241400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_of_other[0]","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:00:19.243429Z","iopub.execute_input":"2022-02-27T19:00:19.243747Z","iopub.status.idle":"2022-02-27T19:00:19.248985Z","shell.execute_reply.started":"2022-02-27T19:00:19.243711Z","shell.execute_reply":"2022-02-27T19:00:19.248205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sr=16000\nvad = malaya_speech.vad.webrtc()\nnoise_wave = []\nnoise_label = []\ni=0\nfor wav in noise_files:\n    if (i<500):\n        samples, sample_rate = librosa.load(noise_path + '/' + wav , sr = 16000)\n    else:\n        samples, sample_rate = librosa.load(noise_audio_path + '/' + wav , sr = 16000)\n    y_= malaya_speech.resample(samples[:15000], sr, 16000)\n    y_ = malaya_speech.astype.float_to_int(y_)\n    frames = malaya_speech.generator.frames(samples, 30, sr)\n    frames_ = list(malaya_speech.generator.frames(y_, 30, 16000, append_ending_trail = False))\n    frames_webrtc = [(frames[no], vad(frame)) for no, frame in enumerate(frames_)]\n    y_ = malaya_speech.combine.without_silent(frames_webrtc)\n    zero = np.zeros(((1*sr+4000)-y_.shape[0]))\n    signal = np.concatenate((y_,zero))\n    noise_wave.append(signal)\n    noise_label.append(label_of_other[0])\n    i=i+1","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-27T19:00:19.250388Z","iopub.execute_input":"2022-02-27T19:00:19.250944Z","iopub.status.idle":"2022-02-27T19:16:00.368931Z","shell.execute_reply.started":"2022-02-27T19:00:19.250905Z","shell.execute_reply":"2022-02-27T19:16:00.368056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"noise_mfcc=[]\nfor wave in noise_wave:\n    mfcc_feat = mfcc(wave , fs, winlen=256/fs, winstep=256/(2*fs), numcep=13, nfilt=26, nfft=256,\n                 lowfreq=0, highfreq=fs/2, preemph=0.97, ceplifter=22, appendEnergy=True, winfunc=np.hamming)\n    mfcc_feat= np.transpose(mfcc_feat)\n    all_mfcc.append(mfcc_feat)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:00.370305Z","iopub.execute_input":"2022-02-27T19:16:00.370646Z","iopub.status.idle":"2022-02-27T19:16:03.236283Z","shell.execute_reply.started":"2022-02-27T19:16:00.370604Z","shell.execute_reply":"2022-02-27T19:16:03.235248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(noise_label)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.240900Z","iopub.execute_input":"2022-02-27T19:16:03.243223Z","iopub.status.idle":"2022-02-27T19:16:03.253080Z","shell.execute_reply.started":"2022-02-27T19:16:03.241411Z","shell.execute_reply":"2022-02-27T19:16:03.252155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.array(all_mfcc).shape)\nprint(np.array(noise_label).shape)\n#d1=np.array(noise_mfcc).shape[1]\n#d2=np.array(noise_mfcc).shape[2]\n#d=d1*d2\n#print(d)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.258046Z","iopub.execute_input":"2022-02-27T19:16:03.258669Z","iopub.status.idle":"2022-02-27T19:16:03.432734Z","shell.execute_reply.started":"2022-02-27T19:16:03.258624Z","shell.execute_reply":"2022-02-27T19:16:03.431273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"op_mfcc=np.array(all_mfcc)\nop_mfcc=op_mfcc.reshape(np.array(all_mfcc).shape[0],-1)\nop_mfcc.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.434134Z","iopub.execute_input":"2022-02-27T19:16:03.434471Z","iopub.status.idle":"2022-02-27T19:16:03.694714Z","shell.execute_reply.started":"2022-02-27T19:16:03.434433Z","shell.execute_reply":"2022-02-27T19:16:03.693721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_label=np.hstack((all_label, noise_label))\nall_label","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.696168Z","iopub.execute_input":"2022-02-27T19:16:03.696510Z","iopub.status.idle":"2022-02-27T19:16:03.707203Z","shell.execute_reply.started":"2022-02-27T19:16:03.696473Z","shell.execute_reply":"2022-02-27T19:16:03.706250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#all_label = all_label.tolist()\n\nle = LabelEncoder()\ny=le.fit_transform(all_label)\nclasses= list(le.classes_)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.708618Z","iopub.execute_input":"2022-02-27T19:16:03.708979Z","iopub.status.idle":"2022-02-27T19:16:03.722056Z","shell.execute_reply.started":"2022-02-27T19:16:03.708943Z","shell.execute_reply":"2022-02-27T19:16:03.721342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model based on ANN** ","metadata":{}},{"cell_type":"code","source":"classes","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.723224Z","iopub.execute_input":"2022-02-27T19:16:03.723761Z","iopub.status.idle":"2022-02-27T19:16:03.729362Z","shell.execute_reply.started":"2022-02-27T19:16:03.723725Z","shell.execute_reply":"2022-02-27T19:16:03.728529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install --upgrade tensorflow\n! pip install --upgrade tensorflow-gpu\n! pip install keras==2.3.1","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:16:03.730652Z","iopub.execute_input":"2022-02-27T19:16:03.731351Z","iopub.status.idle":"2022-02-27T19:18:02.445844Z","shell.execute_reply.started":"2022-02-27T19:16:03.731313Z","shell.execute_reply":"2022-02-27T19:18:02.444916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.optimizers import SGD\nfrom keras.constraints import maxnorm\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Conv2D, Flatten, Dense,Dropout\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:02.447414Z","iopub.execute_input":"2022-02-27T19:18:02.447819Z","iopub.status.idle":"2022-02-27T19:18:02.497535Z","shell.execute_reply.started":"2022-02-27T19:18:02.447775Z","shell.execute_reply":"2022-02-27T19:18:02.496681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=tensorflow.keras.utils.to_categorical(y, num_classes=len(classes), dtype='float32')\ny.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:02.498867Z","iopub.execute_input":"2022-02-27T19:18:02.499476Z","iopub.status.idle":"2022-02-27T19:18:02.507723Z","shell.execute_reply.started":"2022-02-27T19:18:02.499439Z","shell.execute_reply":"2022-02-27T19:18:02.506644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_tr, x_val, y_tr, y_val= train_test_split(op_mfcc,np.array(y),stratify=y,test_size = 0.2,random_state=777,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:02.509368Z","iopub.execute_input":"2022-02-27T19:18:02.509930Z","iopub.status.idle":"2022-02-27T19:18:02.878335Z","shell.execute_reply.started":"2022-02-27T19:18:02.509893Z","shell.execute_reply":"2022-02-27T19:18:02.877441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_tr.shape)\nprint(y_tr.shape)\nprint(x_val.shape)\nprint(y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:02.880709Z","iopub.execute_input":"2022-02-27T19:18:02.881374Z","iopub.status.idle":"2022-02-27T19:18:02.888079Z","shell.execute_reply.started":"2022-02-27T19:18:02.881334Z","shell.execute_reply":"2022-02-27T19:18:02.887087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model Architecture**","metadata":{}},{"cell_type":"code","source":"#Model Architecture\nmodel = Sequential()\nmodel.add(Dense(100, activation='relu', input_shape=(d,), kernel_constraint=maxnorm(3)))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(80, activation='relu', kernel_constraint=maxnorm(3)))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(len(classes), activation='softmax' , kernel_constraint=maxnorm(3)))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:02.889556Z","iopub.execute_input":"2022-02-27T19:18:02.889920Z","iopub.status.idle":"2022-02-27T19:18:03.882240Z","shell.execute_reply.started":"2022-02-27T19:18:02.889882Z","shell.execute_reply":"2022-02-27T19:18:03.881233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensorflow.keras.utils.plot_model(model, 'model.png',show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:03.883550Z","iopub.execute_input":"2022-02-27T19:18:03.883893Z","iopub.status.idle":"2022-02-27T19:18:04.425698Z","shell.execute_reply.started":"2022-02-27T19:18:03.883857Z","shell.execute_reply":"2022-02-27T19:18:04.424720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='categorical_crossentropy',optimizer='adamax',metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:04.427433Z","iopub.execute_input":"2022-02-27T19:18:04.428053Z","iopub.status.idle":"2022-02-27T19:18:04.446164Z","shell.execute_reply.started":"2022-02-27T19:18:04.428009Z","shell.execute_reply":"2022-02-27T19:18:04.445414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10, min_delta=0.0001) \nmc = ModelCheckpoint('best_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:04.447651Z","iopub.execute_input":"2022-02-27T19:18:04.448175Z","iopub.status.idle":"2022-02-27T19:18:04.454004Z","shell.execute_reply.started":"2022-02-27T19:18:04.448136Z","shell.execute_reply":"2022-02-27T19:18:04.453086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#18932\n300*65","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:04.455221Z","iopub.execute_input":"2022-02-27T19:18:04.455622Z","iopub.status.idle":"2022-02-27T19:18:04.465568Z","shell.execute_reply.started":"2022-02-27T19:18:04.455588Z","shell.execute_reply":"2022-02-27T19:18:04.464729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=model.fit(x_tr, y_tr,validation_data=(x_val,y_val), epochs=300, batch_size=65)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:18:04.466722Z","iopub.execute_input":"2022-02-27T19:18:04.467186Z","iopub.status.idle":"2022-02-27T19:22:08.255055Z","shell.execute_reply.started":"2022-02-27T19:18:04.467151Z","shell.execute_reply":"2022-02-27T19:22:08.254152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_score = model.evaluate(x_tr, y_tr, batch_size=12)\nprint(train_score)\n\nprint('----------------Training Complete-----------------')\n\ntest_score = model.evaluate(x_val, y_val, batch_size = 12)\nprint(test_score)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:08.256497Z","iopub.execute_input":"2022-02-27T19:22:08.256851Z","iopub.status.idle":"2022-02-27T19:22:11.299331Z","shell.execute_reply.started":"2022-02-27T19:22:08.256799Z","shell.execute_reply":"2022-02-27T19:22:11.298519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.history.keys()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:11.300684Z","iopub.execute_input":"2022-02-27T19:22:11.301054Z","iopub.status.idle":"2022-02-27T19:22:11.307465Z","shell.execute_reply.started":"2022-02-27T19:22:11.301007Z","shell.execute_reply":"2022-02-27T19:22:11.306537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot\npyplot.plot(history.history['loss'], label='train')\npyplot.plot(history.history['val_loss'], label='test')\npyplot.legend()\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:11.309191Z","iopub.execute_input":"2022-02-27T19:22:11.309958Z","iopub.status.idle":"2022-02-27T19:22:11.479492Z","shell.execute_reply.started":"2022-02-27T19:22:11.309919Z","shell.execute_reply":"2022-02-27T19:22:11.478509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:11.480957Z","iopub.execute_input":"2022-02-27T19:22:11.481310Z","iopub.status.idle":"2022-02-27T19:22:11.631409Z","shell.execute_reply.started":"2022-02-27T19:22:11.481273Z","shell.execute_reply":"2022-02-27T19:22:11.630516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_predict=model.predict(x_val)\nconf_mat=tensorflow.math.confusion_matrix(np.argmax(y_val,axis=1) , np.argmax(y_predict,axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:11.632729Z","iopub.execute_input":"2022-02-27T19:22:11.633123Z","iopub.status.idle":"2022-02-27T19:22:12.436277Z","shell.execute_reply.started":"2022-02-27T19:22:11.633087Z","shell.execute_reply":"2022-02-27T19:22:12.435424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cm = pd.DataFrame(np.array(conf_mat), index = [i for i in classes],\n                  columns = [i for i in classes])\nplt.figure(figsize = (13,7))\nax = sn.heatmap(df_cm, annot=True)\nplt.title(\"Confusion Matrix\", fontsize=20)\nplt.ylabel(\"True Class\"     , fontsize=20)\nplt.xlabel(\"Predicted Class\", fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:12.437641Z","iopub.execute_input":"2022-02-27T19:22:12.437987Z","iopub.status.idle":"2022-02-27T19:22:13.227977Z","shell.execute_reply.started":"2022-02-27T19:22:12.437951Z","shell.execute_reply":"2022-02-27T19:22:13.227213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_val[1].shape\nmodel.predict(x_val[1].reshape((1,d)))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:13.231632Z","iopub.execute_input":"2022-02-27T19:22:13.233687Z","iopub.status.idle":"2022-02-27T19:22:13.292635Z","shell.execute_reply.started":"2022-02-27T19:22:13.233648Z","shell.execute_reply":"2022-02-27T19:22:13.291849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the function that predicts text for the given audio:","metadata":{}},{"cell_type":"code","source":"def predict(audio):\n    print(samples.shape)\n    prob=model.predict(audio)\n    index=np.argmax(prob[0])\n    return classes[index]","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:13.293793Z","iopub.execute_input":"2022-02-27T19:22:13.294140Z","iopub.status.idle":"2022-02-27T19:22:13.299874Z","shell.execute_reply.started":"2022-02-27T19:22:13.294113Z","shell.execute_reply":"2022-02-27T19:22:13.298801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prediction time! Make predictions on the validation data:","metadata":{}},{"cell_type":"code","source":"import random\nindex=random.randint(0,len(x_val)-1)\nprint(index)\nsamples=x_val[index]\nprint(\"Audio:\",classes[np.argmax(y_val[index])])\n#ipd.Audio(np.array(all_wave)[index,:], rate=16000)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:13.301340Z","iopub.execute_input":"2022-02-27T19:22:13.301924Z","iopub.status.idle":"2022-02-27T19:22:13.309726Z","shell.execute_reply.started":"2022-02-27T19:22:13.301882Z","shell.execute_reply":"2022-02-27T19:22:13.308567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Text:\",predict(samples.reshape(1,d)))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:13.311670Z","iopub.execute_input":"2022-02-27T19:22:13.312493Z","iopub.status.idle":"2022-02-27T19:22:13.352286Z","shell.execute_reply.started":"2022-02-27T19:22:13.312447Z","shell.execute_reply":"2022-02-27T19:22:13.351529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model\nmodel.save(\"8_words_other.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-02-27T19:22:13.357423Z","iopub.execute_input":"2022-02-27T19:22:13.357713Z","iopub.status.idle":"2022-02-27T19:22:13.399097Z","shell.execute_reply.started":"2022-02-27T19:22:13.357689Z","shell.execute_reply":"2022-02-27T19:22:13.398328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}