{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import the libraries\n","metadata":{"execution":{"iopub.status.busy":"2022-02-11T15:15:42.09471Z","iopub.execute_input":"2022-02-11T15:15:42.095103Z","iopub.status.idle":"2022-02-11T15:15:42.099724Z","shell.execute_reply.started":"2022-02-11T15:15:42.095016Z","shell.execute_reply":"2022-02-11T15:15:42.098645Z"}}},{"cell_type":"code","source":"!pip install pyunpack\n!pip install patool\n!pip install py7zr\n!pip install sounddevice\n!pip install noisereduce\n!pip install librosa\n! pip install python_speech_features\n! pip install tensorflow==2.4\n! pip install malaya_speech\n! pip install webrtcvad","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:39:51.231206Z","iopub.execute_input":"2022-06-25T22:39:51.231693Z","iopub.status.idle":"2022-06-25T22:43:55.763360Z","shell.execute_reply.started":"2022-06-25T22:39:51.231599Z","shell.execute_reply":"2022-06-25T22:43:55.761628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:43:55.767170Z","iopub.execute_input":"2022-06-25T22:43:55.767735Z","iopub.status.idle":"2022-06-25T22:44:04.853706Z","shell.execute_reply.started":"2022-06-25T22:43:55.767672Z","shell.execute_reply":"2022-06-25T22:44:04.852300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pip update huggingface_hub\n! pip install --upgrade transformers","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:44:04.855764Z","iopub.execute_input":"2022-06-25T22:44:04.857098Z","iopub.status.idle":"2022-06-25T22:44:22.963001Z","shell.execute_reply.started":"2022-06-25T22:44:04.857050Z","shell.execute_reply":"2022-06-25T22:44:22.961802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **importing libraries**","metadata":{}},{"cell_type":"code","source":"pip install git+https://github.com/huggingface/transformers","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:44:22.965656Z","iopub.execute_input":"2022-06-25T22:44:22.966009Z","iopub.status.idle":"2022-06-25T22:45:08.269842Z","shell.execute_reply.started":"2022-06-25T22:44:22.965979Z","shell.execute_reply":"2022-06-25T22:45:08.268367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np                        # linear algebra library\nimport pandas as pd                       # data frames processing\nimport matplotlib.pyplot as plt          # visualization library\nimport seaborn as sn                      # visualization library\n\n\n# audio processing library\nimport librosa                          \nimport IPython.display as ipd            \nfrom scipy.io import wavfile\nimport noisereduce as nr\nfrom malaya_speech import Pipeline\nimport malaya_speech\nfrom python_speech_features import mfcc\nfrom sklearn.preprocessing import LabelEncoder\n\n\n# unpacking the dataset\nfrom py7zr import unpack_7zarchive    \n\n#operating system libraries\nimport shutil\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport tensorflow \nimport os","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:45:08.271794Z","iopub.execute_input":"2022-06-25T22:45:08.272120Z","iopub.status.idle":"2022-06-25T22:45:16.118304Z","shell.execute_reply.started":"2022-06-25T22:45:08.272091Z","shell.execute_reply":"2022-06-25T22:45:16.116832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unpacking the dataset\nshutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)\nshutil.unpack_archive('/kaggle/input/tensorflow-speech-recognition-challenge/train.7z', '/kaggle/working/tensorflow-speech-recognition-challenge/train/')","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:45:16.119974Z","iopub.execute_input":"2022-06-25T22:45:16.120649Z","iopub.status.idle":"2022-06-25T22:55:11.421352Z","shell.execute_reply.started":"2022-06-25T22:45:16.120601Z","shell.execute_reply":"2022-06-25T22:55:11.420444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <center> Implementing the Speech Recognition Model in Python\n**Dataset used for our Speech Recognition Project**\n    \nIt is a set of 10 numbers each is repeated 2000 times with different accents and different back ground conditions. TensorFlow recently released the Speech Commands Datasets. It includes 65,000 one-second long utterances of 30 short words, by thousands of different people. We’ll build a speech recognition system that understands simple spoken commands. <br>    \n    \n__You can download the dataset from__ [here](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge).\n","metadata":{}},{"cell_type":"markdown","source":"**Data Exploration and Visualization**\n\nData Exploration and Visualization helps us to understand the data as well as pre-processing steps in a better way. \n\n","metadata":{}},{"cell_type":"code","source":"train_audio_path = '/kaggle/working/tensorflow-speech-recognition-challenge/train/train/audio/' #path of the training data","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:11.422712Z","iopub.execute_input":"2022-06-25T22:55:11.423010Z","iopub.status.idle":"2022-06-25T22:55:11.428030Z","shell.execute_reply.started":"2022-06-25T22:55:11.422982Z","shell.execute_reply":"2022-06-25T22:55:11.426987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Accessing a sample file in data**","metadata":{}},{"cell_type":"code","source":"samples, sample_rate = librosa.load(train_audio_path+'on/5a3712c9_nohash_1.wav', sr = 16000)  # loading a sample to be explored carefully\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + '../input/train/audio/on/0a7c2a8d_nohash_0.wav')\nax1.set_xlabel('time')\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:11.429449Z","iopub.execute_input":"2022-06-25T22:55:11.430001Z","iopub.status.idle":"2022-06-25T22:55:11.674295Z","shell.execute_reply.started":"2022-06-25T22:55:11.429972Z","shell.execute_reply":"2022-06-25T22:55:11.672634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fs=16000\nipd.Audio(samples, rate=fs)  #listen to audio file before noise reduction\nprint(\"Sample Rate:\",fs)\nsr=fs","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:11.675880Z","iopub.execute_input":"2022-06-25T22:55:11.677365Z","iopub.status.idle":"2022-06-25T22:55:11.684292Z","shell.execute_reply.started":"2022-06-25T22:55:11.677319Z","shell.execute_reply":"2022-06-25T22:55:11.683177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Noise Reduction\ntime = np.linspace(0, len(samples - 1) / fs, len(samples - 1))\nreduced_noise1 = nr.reduce_noise(y=samples, sr=fs,stationary=True)\nplt.plot(time, reduced_noise1)  # plot in seconds\nplt.xlabel(\"Time [seconds]\")\nplt.ylabel(\"Voice amplitude\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:11.689642Z","iopub.execute_input":"2022-06-25T22:55:11.690151Z","iopub.status.idle":"2022-06-25T22:55:12.613949Z","shell.execute_reply.started":"2022-06-25T22:55:11.690116Z","shell.execute_reply":"2022-06-25T22:55:12.612538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(reduced_noise1, rate=sample_rate)  #listen to audio file after noise reduction","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:12.617428Z","iopub.execute_input":"2022-06-25T22:55:12.617795Z","iopub.status.idle":"2022-06-25T22:55:12.626550Z","shell.execute_reply.started":"2022-06-25T22:55:12.617764Z","shell.execute_reply":"2022-06-25T22:55:12.624425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Silence Removal\nvad = malaya_speech.vad.webrtc()\ny=reduced_noise1\ny_= malaya_speech.resample(y, sr, 16000)\ny_ = malaya_speech.astype.float_to_int(y_)\nframes = malaya_speech.generator.frames(y, 30, sr)\nframes_ = list(malaya_speech.generator.frames(y_, 30, 16000, append_ending_trail = False))\nframes_webrtc = [(frames[no], vad(frame)) for no, frame in enumerate(frames_)]\ny_ = malaya_speech.combine.without_silent(frames_webrtc)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:12.628055Z","iopub.execute_input":"2022-06-25T22:55:12.628985Z","iopub.status.idle":"2022-06-25T22:55:12.828083Z","shell.execute_reply.started":"2022-06-25T22:55:12.628941Z","shell.execute_reply":"2022-06-25T22:55:12.827263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipd.Audio(y_, rate = sr )     #listen to audio file after noise reduction","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:12.829317Z","iopub.execute_input":"2022-06-25T22:55:12.830217Z","iopub.status.idle":"2022-06-25T22:55:12.837013Z","shell.execute_reply.started":"2022-06-25T22:55:12.830169Z","shell.execute_reply":"2022-06-25T22:55:12.836053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# padding signal with zeros\nzero = np.zeros((1*sr-y_.shape[0]))\nsignal = np.concatenate((y_,zero))\nsignal.shape\ntime = np.linspace(0, len(signal - 1) / fs, len(signal - 1))","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:12.838398Z","iopub.execute_input":"2022-06-25T22:55:12.838714Z","iopub.status.idle":"2022-06-25T22:55:12.849439Z","shell.execute_reply.started":"2022-06-25T22:55:12.838687Z","shell.execute_reply":"2022-06-25T22:55:12.848385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualization of Audio signal in time series domain**\n\nNow, we’ll visualize the audio signal in the time series domain:","metadata":{}},{"cell_type":"code","source":"plt.plot(time,signal)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:12.851202Z","iopub.execute_input":"2022-06-25T22:55:12.851545Z","iopub.status.idle":"2022-06-25T22:55:13.035897Z","shell.execute_reply.started":"2022-06-25T22:55:12.851497Z","shell.execute_reply":"2022-06-25T22:55:13.035124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=os.listdir(train_audio_path)     #Extracting labels to determine classes","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:13.037184Z","iopub.execute_input":"2022-06-25T22:55:13.037663Z","iopub.status.idle":"2022-06-25T22:55:13.042808Z","shell.execute_reply.started":"2022-06-25T22:55:13.037629Z","shell.execute_reply":"2022-06-25T22:55:13.041653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Distribution of the Data set**","metadata":{}},{"cell_type":"code","source":"#find count of each label and plot bar graph\nno_of_recordings=[]\nfor label in labels:\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label) if f.endswith('.wav')]\n    no_of_recordings.append(len(waves))\n    \n#plot\nplt.figure(figsize=(30,5))\nindex = np.arange(len(labels))\nplt.bar(index, no_of_recordings)\nplt.xlabel('Commands', fontsize=12)\nplt.ylabel('No of recordings', fontsize=12)\nplt.xticks(index, labels, fontsize=15, rotation=60)\nplt.title('No. of recordings for each command')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:13.044152Z","iopub.execute_input":"2022-06-25T22:55:13.044631Z","iopub.status.idle":"2022-06-25T22:55:13.434901Z","shell.execute_reply.started":"2022-06-25T22:55:13.044602Z","shell.execute_reply":"2022-06-25T22:55:13.433799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Words used as Classes for data set","metadata":{}},{"cell_type":"code","source":"labels=[\"down\",\"left\",\"right\",\"stop\",\"up\"]","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:13.436078Z","iopub.execute_input":"2022-06-25T22:55:13.436409Z","iopub.status.idle":"2022-06-25T22:55:13.442422Z","shell.execute_reply.started":"2022-06-25T22:55:13.436379Z","shell.execute_reply":"2022-06-25T22:55:13.441252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Preprocessing the audio waves**\n\nlet us read the audio waves and use the below-preprocessing steps :\n\n* Noise Reduction\n* Silence Removal\n* Extracting MFCCs\n","metadata":{}},{"cell_type":"code","source":"sr=16000    # sample rate\nvad = malaya_speech.vad.webrtc()\nall_wave = []     #intitialize array to stack wave files of the whole data set in it \nall_label = []    #intitialize array to stack label of wave files of the whole data set in it \nfor label in labels:\n    print(label)\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label) if f.endswith('.wav')] # access on each file\n    for wav in waves:\n        samples, sample_rate = librosa.load(train_audio_path + '/' + label + '/' + wav, sr = 16000)\n        samples = nr.reduce_noise(y=samples, sr=sr,stationary=True)  #noise reduction\n        y_= malaya_speech.resample(samples, sr, 16000)               # silence removal\n        y_ = malaya_speech.astype.float_to_int(y_)\n        frames = malaya_speech.generator.frames(samples, 30, sr)\n        frames_ = list(malaya_speech.generator.frames(y_, 30, 16000, append_ending_trail = False))\n        frames_webrtc = [(frames[no], vad(frame)) for no, frame in enumerate(frames_)]\n        y_ = malaya_speech.combine.without_silent(frames_webrtc)\n        zero = np.zeros(((1*sr+4000)-y_.shape[0]))                 \n        signal = np.concatenate((y_,zero))     # concatenation with zeros to adust length of the vector\n        all_wave.append(signal)     #append waves one by one \n        all_label.append(label)     #append corresponding label one by one","metadata":{"execution":{"iopub.status.busy":"2022-06-25T22:55:13.444152Z","iopub.execute_input":"2022-06-25T22:55:13.445126Z","iopub.status.idle":"2022-06-25T23:01:15.271755Z","shell.execute_reply.started":"2022-06-25T22:55:13.445082Z","shell.execute_reply":"2022-06-25T23:01:15.270498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape of waves array\",np.array(all_wave).shape)\nprint(\"shape of labels array\",np.array(all_label).shape)\n\n#Inspecting random sample\ntime = np.linspace(0, len(signal - 1) / fs, len(signal - 1))\nplt.plot(time,np.array(all_wave)[2000,:])\nprint(np.array(all_label)[2000])\nipd.Audio(np.array(all_wave)[2000,:], rate = sr )","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:01:15.273231Z","iopub.execute_input":"2022-06-25T23:01:15.273679Z","iopub.status.idle":"2022-06-25T23:01:17.955798Z","shell.execute_reply.started":"2022-06-25T23:01:15.273638Z","shell.execute_reply":"2022-06-25T23:01:17.954841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extracting MFCCs","metadata":{}},{"cell_type":"code","source":"all_mfcc=[]    #intitialize array to stack MFCCs of the whole data set in it\nfor wave in all_wave:\n    i=0\n    mfcc_feat = mfcc(wave , fs, winlen=256/fs, winstep=256/(2*fs), numcep=13, nfilt=26, nfft=256,\n                 lowfreq=0, highfreq=fs/2, preemph=0.97, ceplifter=22, appendEnergy=True, winfunc=np.hamming)\n    mfcc_feat= np.transpose(mfcc_feat)\n    all_mfcc.append(mfcc_feat)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:01:17.957116Z","iopub.execute_input":"2022-06-25T23:01:17.958012Z","iopub.status.idle":"2022-06-25T23:01:53.176743Z","shell.execute_reply.started":"2022-06-25T23:01:17.957973Z","shell.execute_reply":"2022-06-25T23:01:53.175402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape of MFCCs\",np.array(all_mfcc).shape)\nprint(\"shape of Corresponding lables\",np.array(all_label).shape)\n\n# dimensions of the data\nd1=np.array(all_mfcc).shape[1]\nd2=np.array(all_mfcc).shape[2]\nd=d1*d2","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:01:53.178401Z","iopub.execute_input":"2022-06-25T23:01:53.179492Z","iopub.status.idle":"2022-06-25T23:01:53.471582Z","shell.execute_reply.started":"2022-06-25T23:01:53.179441Z","shell.execute_reply":"2022-06-25T23:01:53.470342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"op_mfcc=np.array(all_mfcc)     # transform list to array to be fed to the model\nop_mfcc=op_mfcc.reshape(np.array(all_mfcc).shape[0],-1)   # adjust shape of the samples\nop_mfcc.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:01:53.472984Z","iopub.execute_input":"2022-06-25T23:01:53.473307Z","iopub.status.idle":"2022-06-25T23:01:53.643029Z","shell.execute_reply.started":"2022-06-25T23:01:53.473278Z","shell.execute_reply":"2022-06-25T23:01:53.641883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One hot encoding","metadata":{}},{"cell_type":"code","source":"#all_label = all_label.tolist()\nle = LabelEncoder()\ny=le.fit_transform(all_label)\nclasses= list(le.classes_)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:01:53.644539Z","iopub.execute_input":"2022-06-25T23:01:53.644860Z","iopub.status.idle":"2022-06-25T23:01:53.653453Z","shell.execute_reply.started":"2022-06-25T23:01:53.644832Z","shell.execute_reply":"2022-06-25T23:01:53.652118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model based on ANN** ","metadata":{}},{"cell_type":"code","source":"! pip install --upgrade tensorflow\n! pip install --upgrade tensorflow-gpu\n! pip install keras==2.3.1","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:01:53.655083Z","iopub.execute_input":"2022-06-25T23:01:53.655433Z","iopub.status.idle":"2022-06-25T23:04:57.590065Z","shell.execute_reply.started":"2022-06-25T23:01:53.655404Z","shell.execute_reply":"2022-06-25T23:04:57.588520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing model libraries from keras\nfrom keras.optimizers import SGD\nfrom keras.constraints import maxnorm\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Flatten, Dense,Dropout\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom sklearn.model_selection import train_test_split\n","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:57.593153Z","iopub.execute_input":"2022-06-25T23:04:57.593737Z","iopub.status.idle":"2022-06-25T23:04:57.659109Z","shell.execute_reply.started":"2022-06-25T23:04:57.593679Z","shell.execute_reply":"2022-06-25T23:04:57.657926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=tensorflow.keras.utils.to_categorical(y, num_classes=len(labels), dtype='float32')  # one hot encoded varibales to categeorical values\ny.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:57.660432Z","iopub.execute_input":"2022-06-25T23:04:57.660741Z","iopub.status.idle":"2022-06-25T23:04:57.669783Z","shell.execute_reply.started":"2022-06-25T23:04:57.660714Z","shell.execute_reply":"2022-06-25T23:04:57.668437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_tr, x_val, y_tr, y_val= train_test_split(op_mfcc,np.array(y),stratify=y,test_size = 0.2,random_state=777,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:57.676569Z","iopub.execute_input":"2022-06-25T23:04:57.677094Z","iopub.status.idle":"2022-06-25T23:04:57.949301Z","shell.execute_reply.started":"2022-06-25T23:04:57.677060Z","shell.execute_reply":"2022-06-25T23:04:57.947830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inspect shapes of each set\nprint(\"shape of training samples\",x_tr.shape)\nprint(\"shape of training labels\",y_tr.shape)\nprint(\"shape of test samples\",x_val.shape)\nprint(\"shape of test labels\",y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:57.951045Z","iopub.execute_input":"2022-06-25T23:04:57.951438Z","iopub.status.idle":"2022-06-25T23:04:57.958057Z","shell.execute_reply.started":"2022-06-25T23:04:57.951405Z","shell.execute_reply":"2022-06-25T23:04:57.956804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model Architecture**","metadata":{}},{"cell_type":"code","source":"#Model Architecture\nmodel = Sequential()\nmodel.add(Dense(100, activation='relu', input_shape=(d,), kernel_constraint=maxnorm(3)))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(80, activation='relu', kernel_constraint=maxnorm(3)))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(len(classes), activation='softmax' , kernel_constraint=maxnorm(3)))","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:57.959475Z","iopub.execute_input":"2022-06-25T23:04:57.960403Z","iopub.status.idle":"2022-06-25T23:04:58.628190Z","shell.execute_reply.started":"2022-06-25T23:04:57.960371Z","shell.execute_reply":"2022-06-25T23:04:58.627105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tensorflow.keras.utils.plot_model(model, 'model.png',show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:58.629972Z","iopub.execute_input":"2022-06-25T23:04:58.630389Z","iopub.status.idle":"2022-06-25T23:04:59.932055Z","shell.execute_reply.started":"2022-06-25T23:04:58.630350Z","shell.execute_reply":"2022-06-25T23:04:59.929153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='categorical_crossentropy',optimizer='adamax',metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:59.933742Z","iopub.execute_input":"2022-06-25T23:04:59.934057Z","iopub.status.idle":"2022-06-25T23:04:59.955480Z","shell.execute_reply.started":"2022-06-25T23:04:59.934027Z","shell.execute_reply":"2022-06-25T23:04:59.954377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10, min_delta=0.0001) \nmc = ModelCheckpoint('best_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:59.958667Z","iopub.execute_input":"2022-06-25T23:04:59.959054Z","iopub.status.idle":"2022-06-25T23:04:59.964434Z","shell.execute_reply.started":"2022-06-25T23:04:59.959023Z","shell.execute_reply":"2022-06-25T23:04:59.963356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=model.fit(x_tr, y_tr,validation_data=(x_val,y_val), epochs=300, batch_size=65)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:04:59.966070Z","iopub.execute_input":"2022-06-25T23:04:59.966386Z","iopub.status.idle":"2022-06-25T23:08:03.961063Z","shell.execute_reply.started":"2022-06-25T23:04:59.966358Z","shell.execute_reply":"2022-06-25T23:08:03.959887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Evaluation","metadata":{}},{"cell_type":"code","source":"train_score = model.evaluate(x_tr, y_tr, batch_size=12)\nprint(train_score)\n\nprint('----------------Training Complete-----------------')\n\ntest_score = model.evaluate(x_val, y_val, batch_size = 12)\nprint(test_score)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:08:03.963739Z","iopub.execute_input":"2022-06-25T23:08:03.964969Z","iopub.status.idle":"2022-06-25T23:08:05.235674Z","shell.execute_reply.started":"2022-06-25T23:08:03.964923Z","shell.execute_reply":"2022-06-25T23:08:05.234143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.history.keys()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:08:05.237129Z","iopub.execute_input":"2022-06-25T23:08:05.238069Z","iopub.status.idle":"2022-06-25T23:08:05.245082Z","shell.execute_reply.started":"2022-06-25T23:08:05.238024Z","shell.execute_reply":"2022-06-25T23:08:05.243874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'], label='train')  # losses learning curve of training set.\nplt.plot(history.history['val_loss'], label='test') # losses learning curve of validation set.\nplt.legend()\nplt.title(\"losses learning curves\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:08:05.246632Z","iopub.execute_input":"2022-06-25T23:08:05.246938Z","iopub.status.idle":"2022-06-25T23:08:05.453226Z","shell.execute_reply.started":"2022-06-25T23:08:05.246909Z","shell.execute_reply":"2022-06-25T23:08:05.451963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['accuracy'])      # Accuracy learning curve of training set.\nplt.plot(history.history['val_accuracy'])  # Accuracy learning curve of validation set.\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.title(\"Accuracy learning curves\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:08:05.454818Z","iopub.execute_input":"2022-06-25T23:08:05.455156Z","iopub.status.idle":"2022-06-25T23:08:05.643579Z","shell.execute_reply.started":"2022-06-25T23:08:05.455126Z","shell.execute_reply":"2022-06-25T23:08:05.642573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **confusion matrix**","metadata":{}},{"cell_type":"code","source":"y_predict=model.predict(x_val)\nconf_mat=tensorflow.math.confusion_matrix(np.argmax(y_val,axis=1) , np.argmax(y_predict,axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:08:05.644925Z","iopub.execute_input":"2022-06-25T23:08:05.645217Z","iopub.status.idle":"2022-06-25T23:08:06.179764Z","shell.execute_reply.started":"2022-06-25T23:08:05.645191Z","shell.execute_reply":"2022-06-25T23:08:06.178585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cm = pd.DataFrame(np.array(conf_mat), index = [i for i in classes],\n                  columns = [i for i in classes])\nplt.figure(figsize = (13,7))\nax = sn.heatmap(df_cm, annot=True)\nplt.title(\"Confusion Matrix\", fontsize=20)\nplt.ylabel(\"True Class\"     , fontsize=20)\nplt.xlabel(\"Predicted Class\", fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:08:06.181026Z","iopub.execute_input":"2022-06-25T23:08:06.181329Z","iopub.status.idle":"2022-06-25T23:08:06.485162Z","shell.execute_reply.started":"2022-06-25T23:08:06.181303Z","shell.execute_reply":"2022-06-25T23:08:06.483815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test random samples to check model performance","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_val[1].shape\nmodel.predict(x_val[1].reshape((1,d)))","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:09:47.924170Z","iopub.execute_input":"2022-06-25T23:09:47.924681Z","iopub.status.idle":"2022-06-25T23:09:47.974865Z","shell.execute_reply.started":"2022-06-25T23:09:47.924644Z","shell.execute_reply":"2022-06-25T23:09:47.974025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the function that predicts text for the given audio:","metadata":{}},{"cell_type":"code","source":"def predict(audio):\n    print(samples.shape)\n    prob=model.predict(audio)\n    index=np.argmax(prob[0])\n    return classes[index]","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:09:50.932077Z","iopub.execute_input":"2022-06-25T23:09:50.932732Z","iopub.status.idle":"2022-06-25T23:09:50.938901Z","shell.execute_reply.started":"2022-06-25T23:09:50.932680Z","shell.execute_reply":"2022-06-25T23:09:50.938045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prediction time! Make predictions on the validation data:","metadata":{}},{"cell_type":"code","source":"import random\nindex=random.randint(0,len(x_val)-1)\nprint(index)\nsamples=x_val[index]\nprint(\"Audio:\",classes[np.argmax(y_val[index])])\n#ipd.Audio(np.array(all_wave)[index,:], rate=16000)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:09:52.452150Z","iopub.execute_input":"2022-06-25T23:09:52.453373Z","iopub.status.idle":"2022-06-25T23:09:52.459080Z","shell.execute_reply.started":"2022-06-25T23:09:52.453330Z","shell.execute_reply":"2022-06-25T23:09:52.458262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Text:\",predict(samples.reshape(1,d)))","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:09:53.188209Z","iopub.execute_input":"2022-06-25T23:09:53.189439Z","iopub.status.idle":"2022-06-25T23:09:53.234847Z","shell.execute_reply.started":"2022-06-25T23:09:53.189391Z","shell.execute_reply":"2022-06-25T23:09:53.233848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model\nmodel.save(\"SR_MODEL.h5\")    #loading model to be tested locally.","metadata":{"execution":{"iopub.status.busy":"2022-06-25T23:09:53.988965Z","iopub.execute_input":"2022-06-25T23:09:53.989604Z","iopub.status.idle":"2022-06-25T23:09:54.041059Z","shell.execute_reply.started":"2022-06-25T23:09:53.989568Z","shell.execute_reply":"2022-06-25T23:09:54.039830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}