{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I strongly recommend to go through the article [here](https://www.analyticsvidhya.com/blog/2019/07/learn-build-first-speech-to-text-model-python/) to understand the basics of signal processing prior implementing the speech to text.\n\n**Understanding the Problem Statement for our Speech-to-Text Project**\n\nO TensorFlow lançou recentemente os conjuntos de dados de comandos de fala. Inclui 65.000 enunciados de um segundo de 30 palavras curtas, por milhares de pessoas diferentes. Construiremos um sistema de reconhecimento de fala que entenda comandos falados simples.\n\n**Import the libraries**\n\nFirst, import all the necessary libraries into our notebook. LibROSA and SciPy are the Python libraries used for processing audio signals.","metadata":{}},{"cell_type":"code","source":"import os\nimport librosa\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom scipy.io import wavfile\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:45:35.641393Z","iopub.execute_input":"2022-10-06T12:45:35.64183Z","iopub.status.idle":"2022-10-06T12:45:37.725735Z","shell.execute_reply.started":"2022-10-06T12:45:35.641798Z","shell.execute_reply":"2022-10-06T12:45:37.724573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:13:46.858187Z","iopub.execute_input":"2022-10-06T12:13:46.859456Z","iopub.status.idle":"2022-10-06T12:13:46.871158Z","shell.execute_reply.started":"2022-10-06T12:13:46.859403Z","shell.execute_reply":"2022-10-06T12:13:46.86999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Exploração e visualização de dados**\n\nA Exploração e Visualização de Dados nos ajuda a entender os dados, bem como as etapas de pré-processamento de uma maneira melhor.\n\n**Visualização do sinal de áudio no domínio da série temporal**\n\nAgora, vamos visualizar o sinal de áudio no domínio da série temporal:","metadata":{}},{"cell_type":"code","source":"import os\nfrom os.path import isdir, join\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:33:47.24916Z","iopub.execute_input":"2022-10-06T12:33:47.249582Z","iopub.status.idle":"2022-10-06T12:33:47.255106Z","shell.execute_reply.started":"2022-10-06T12:33:47.249551Z","shell.execute_reply":"2022-10-06T12:33:47.253938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyunpack\n!pip install patool","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:38:17.199489Z","iopub.execute_input":"2022-10-06T12:38:17.199909Z","iopub.status.idle":"2022-10-06T12:38:44.528825Z","shell.execute_reply.started":"2022-10-06T12:38:17.199873Z","shell.execute_reply":"2022-10-06T12:38:44.527581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pyunpack import Archive\nimport shutil\nif not os.path.exists('/kaggle/working/train/'):\n    os.makedirs('/kaggle/working/train/')\nArchive('../input/tensorflow-speech-recognition-challenge/train.7z').extractall('/kaggle/working/train/')\nfor dirname, _, filenames in os.walk('/kaggle/working/train/'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:40:15.462463Z","iopub.execute_input":"2022-10-06T12:40:15.462901Z","iopub.status.idle":"2022-10-06T12:42:12.040342Z","shell.execute_reply.started":"2022-10-06T12:40:15.462865Z","shell.execute_reply":"2022-10-06T12:42:12.037534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install librosa","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:45:09.348934Z","iopub.execute_input":"2022-10-06T12:45:09.349454Z","iopub.status.idle":"2022-10-06T12:45:20.348859Z","shell.execute_reply.started":"2022-10-06T12:45:09.349416Z","shell.execute_reply":"2022-10-06T12:45:20.346939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_audio_path = './train/train/audio/'\nsamples, sample_rate = librosa.load(train_audio_path+'yes/0a7c2a8d_nohash_0.wav', sr = 16000)\nfig = plt.figure(figsize=(14, 8))\nax1 = fig.add_subplot(211)\nax1.set_title('Raw wave of ' + '../input/train/audio/yes/0a7c2a8d_nohash_0.wav')\nax1.set_xlabel('time')\nax1.set_ylabel('Amplitude')\nax1.plot(np.linspace(0, sample_rate/len(samples), sample_rate), samples)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:51:40.572543Z","iopub.execute_input":"2022-10-06T12:51:40.57307Z","iopub.status.idle":"2022-10-06T12:51:40.8094Z","shell.execute_reply.started":"2022-10-06T12:51:40.57303Z","shell.execute_reply":"2022-10-06T12:51:40.808511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Taxa de amostragem**\n","metadata":{}},{"cell_type":"code","source":"ipd.Audio(samples, rate=sample_rate)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:46:03.845879Z","iopub.execute_input":"2022-10-06T12:46:03.847151Z","iopub.status.idle":"2022-10-06T12:46:03.856143Z","shell.execute_reply.started":"2022-10-06T12:46:03.847096Z","shell.execute_reply":"2022-10-06T12:46:03.855037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_rate)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:46:06.359837Z","iopub.execute_input":"2022-10-06T12:46:06.36026Z","iopub.status.idle":"2022-10-06T12:46:06.365431Z","shell.execute_reply.started":"2022-10-06T12:46:06.360225Z","shell.execute_reply":"2022-10-06T12:46:06.364357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reamostragem**\n\nPelo exposto, podemos entender que a taxa de amostragem do sinal é de 16.000 hz. Vamos reamostrar para 8000 hz, já que a maioria das frequências relacionadas à fala está presente em 8000z","metadata":{}},{"cell_type":"code","source":"samples = librosa.resample(samples, sample_rate, 8000)\nipd.Audio(samples, rate=8000)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:46:08.91998Z","iopub.execute_input":"2022-10-06T12:46:08.92039Z","iopub.status.idle":"2022-10-06T12:46:09.841331Z","shell.execute_reply.started":"2022-10-06T12:46:08.920358Z","shell.execute_reply":"2022-10-06T12:46:09.84019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Agora, vamos entender o número de gravações para cada comando de voz:","metadata":{}},{"cell_type":"code","source":"labels=os.listdir(train_audio_path)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:51:47.568856Z","iopub.execute_input":"2022-10-06T12:51:47.569313Z","iopub.status.idle":"2022-10-06T12:51:47.574779Z","shell.execute_reply.started":"2022-10-06T12:51:47.569271Z","shell.execute_reply":"2022-10-06T12:51:47.573606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#find count of each label and plot bar graph\nno_of_recordings=[]\nfor label in labels:\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label)  if f.endswith('.wav')]\n    no_of_recordings.append(len(waves))\n    \n#plot\nplt.figure(figsize=(30,5))\nindex = np.arange(len(labels))\nplt.bar(index, no_of_recordings)\nplt.xlabel('Commands', fontsize=12)\nplt.ylabel('No of recordings', fontsize=12)\nplt.xticks(index, labels, fontsize=15, rotation=60)\nplt.title('No. of recordings for each command')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:52:14.893274Z","iopub.execute_input":"2022-10-06T12:52:14.893703Z","iopub.status.idle":"2022-10-06T12:52:15.291718Z","shell.execute_reply.started":"2022-10-06T12:52:14.893669Z","shell.execute_reply":"2022-10-06T12:52:15.290504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=[\"yes\", \"no\", \"up\", \"down\", \"left\", \"right\", \"on\", \"off\", \"stop\", \"go\"]","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:52:22.093444Z","iopub.execute_input":"2022-10-06T12:52:22.093838Z","iopub.status.idle":"2022-10-06T12:52:22.099583Z","shell.execute_reply.started":"2022-10-06T12:52:22.093808Z","shell.execute_reply":"2022-10-06T12:52:22.098298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Duração das gravações** \n\nDistribuição da duração das gravações:","metadata":{}},{"cell_type":"code","source":"duration_of_recordings=[]\nfor label in labels:\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label) if f.endswith('.wav')]\n    for wav in waves:\n        sample_rate, samples = wavfile.read(train_audio_path + '/' + label + '/' + wav)\n        duration_of_recordings.append(float(len(samples)/sample_rate))\n    \nplt.hist(np.array(duration_of_recordings))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:52:27.260589Z","iopub.execute_input":"2022-10-06T12:52:27.261009Z","iopub.status.idle":"2022-10-06T12:52:34.747445Z","shell.execute_reply.started":"2022-10-06T12:52:27.260973Z","shell.execute_reply":"2022-10-06T12:52:34.746246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pré-processamento das ondas de áudio**\n\nNa parte de exploração de dados anterior, vimos que a duração de algumas gravações é inferior a 1 segundo e a taxa de amostragem é muito alta. Então, vamos ler as ondas de áudio e usar as etapas de pré-processamento abaixo para lidar com isso.\n\nAqui estão os dois passos que seguiremos:\n\n- Reamostragem\n- Removendo comandos mais curtos de menos de 1 segundo\n\nVamos definir essas etapas de pré-processamento no trecho de código abaixo:","metadata":{}},{"cell_type":"code","source":"all_wave = []\nall_label = []\nfor label in labels:\n    print(label)\n    waves = [f for f in os.listdir(train_audio_path + '/'+ label) if f.endswith('.wav')]\n    for wav in waves:\n        samples, sample_rate = librosa.load(train_audio_path + '/' + label + '/' + wav, sr = 16000)\n        samples = librosa.resample(samples, sample_rate, 8000)\n        if(len(samples)== 8000) : \n            all_wave.append(samples)\n            all_label.append(label)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T12:53:03.614546Z","iopub.execute_input":"2022-10-06T12:53:03.615591Z","iopub.status.idle":"2022-10-06T13:00:33.834105Z","shell.execute_reply.started":"2022-10-06T12:53:03.615551Z","shell.execute_reply":"2022-10-06T13:00:33.832733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converta os rótulos de saída em inteiros codificados:","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\ny=le.fit_transform(all_label)\nclasses= list(le.classes_)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:02:55.971165Z","iopub.execute_input":"2022-10-06T13:02:55.971597Z","iopub.status.idle":"2022-10-06T13:02:55.983391Z","shell.execute_reply.started":"2022-10-06T13:02:55.971562Z","shell.execute_reply":"2022-10-06T13:02:55.982154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Agora, converta os rótulos codificados inteiros em um vetor one-hot, pois é um problema de multiclassificação:","metadata":{}},{"cell_type":"code","source":"from keras.utils import np_utils\ny=np_utils.to_categorical(y, num_classes=len(labels))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:02:59.338565Z","iopub.execute_input":"2022-10-06T13:02:59.338972Z","iopub.status.idle":"2022-10-06T13:02:59.344889Z","shell.execute_reply.started":"2022-10-06T13:02:59.338926Z","shell.execute_reply":"2022-10-06T13:02:59.343994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Remodele a matriz 2D para 3D, pois a entrada para o conv1d deve ser uma matriz 3D:","metadata":{}},{"cell_type":"code","source":"all_wave = np.array(all_wave).reshape(-1,8000,1)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:03:03.330058Z","iopub.execute_input":"2022-10-06T13:03:03.330498Z","iopub.status.idle":"2022-10-06T13:03:03.654482Z","shell.execute_reply.started":"2022-10-06T13:03:03.330463Z","shell.execute_reply":"2022-10-06T13:03:03.653452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Divida em conjunto de treinamento e validação**\n\nEm seguida, vamos treinar o modelo em 80% dos dados e validar nos 20% restantes:\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_tr, x_val, y_tr, y_val = train_test_split(np.array(all_wave),np.array(y),stratify=y,test_size = 0.2,random_state=777,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:03:07.318804Z","iopub.execute_input":"2022-10-06T13:03:07.319616Z","iopub.status.idle":"2022-10-06T13:03:08.093593Z","shell.execute_reply.started":"2022-10-06T13:03:07.319575Z","shell.execute_reply":"2022-10-06T13:03:08.092333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Arquitetura de modelo para este problema**\n\nConstruiremos o modelo de fala para texto usando conv1d. Conv1d é uma rede neural convolucional que realiza a convolução ao longo de apenas uma dimensão.","metadata":{}},{"cell_type":"markdown","source":"**Construção do modelo**\n\nVamos implementar o modelo usando a API funcional Keras.","metadata":{}},{"cell_type":"code","source":"from keras.layers import Dense, Dropout, Flatten, Conv1D, Input, MaxPooling1D\nfrom keras.models import Model\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras import backend as K\nK.clear_session()\n\ninputs = Input(shape=(8000,1))\n\n#First Conv1D layer\nconv = Conv1D(8,13, padding='valid', activation='relu', strides=1)(inputs)\nconv = MaxPooling1D(3)(conv)\nconv = Dropout(0.3)(conv)\n\n#Second Conv1D layer\nconv = Conv1D(16, 11, padding='valid', activation='relu', strides=1)(conv)\nconv = MaxPooling1D(3)(conv)\nconv = Dropout(0.3)(conv)\n\n#Third Conv1D layer\nconv = Conv1D(32, 9, padding='valid', activation='relu', strides=1)(conv)\nconv = MaxPooling1D(3)(conv)\nconv = Dropout(0.3)(conv)\n\n#Fourth Conv1D layer\nconv = Conv1D(64, 7, padding='valid', activation='relu', strides=1)(conv)\nconv = MaxPooling1D(3)(conv)\nconv = Dropout(0.3)(conv)\n\n#Flatten layer\nconv = Flatten()(conv)\n\n#Dense Layer 1\nconv = Dense(256, activation='relu')(conv)\nconv = Dropout(0.3)(conv)\n\n#Dense Layer 2\nconv = Dense(128, activation='relu')(conv)\nconv = Dropout(0.3)(conv)\n\noutputs = Dense(len(labels), activation='softmax')(conv)\n\nmodel = Model(inputs, outputs)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:03:12.663148Z","iopub.execute_input":"2022-10-06T13:03:12.663569Z","iopub.status.idle":"2022-10-06T13:03:12.898162Z","shell.execute_reply.started":"2022-10-06T13:03:12.663537Z","shell.execute_reply":"2022-10-06T13:03:12.897179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defina a função de perda como entropia cruzada categórica, pois é um problema de multiclassificação:","metadata":{}},{"cell_type":"code","source":"model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:03:34.03184Z","iopub.execute_input":"2022-10-06T13:03:34.033238Z","iopub.status.idle":"2022-10-06T13:03:34.048798Z","shell.execute_reply.started":"2022-10-06T13:03:34.033176Z","shell.execute_reply":"2022-10-06T13:03:34.047739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Paradas antecipadas e checkpoints de modelo são os retornos de chamada para interromper o treinamento da rede neural no momento certo e salvar o melhor modelo após cada época:","metadata":{}},{"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10, min_delta=0.0001) \nmc = ModelCheckpoint('best_model.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:06:37.310787Z","iopub.execute_input":"2022-10-06T14:06:37.311475Z","iopub.status.idle":"2022-10-06T14:06:37.317214Z","shell.execute_reply.started":"2022-10-06T14:06:37.311438Z","shell.execute_reply":"2022-10-06T14:06:37.315854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Vamos treinar o modelo em um tamanho de lote de 32 e avaliar o desempenho no conjunto de validação:","metadata":{}},{"cell_type":"code","source":"history=model.fit(x_tr, y_tr ,epochs=100, callbacks=[es,mc], batch_size=32, validation_data=(x_val,y_val))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:03:40.831239Z","iopub.execute_input":"2022-10-06T13:03:40.831735Z","iopub.status.idle":"2022-10-06T13:34:28.047177Z","shell.execute_reply.started":"2022-10-06T13:03:40.831688Z","shell.execute_reply":"2022-10-06T13:34:28.046223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Gráfico de diagnóstico**\n\nVou me basear na visualização novamente para entender o desempenho do modelo ao longo de um período de tempo:","metadata":{}},{"cell_type":"code","source":"from matplotlib import pyplot\npyplot.plot(history.history['loss'], label='train')\npyplot.plot(history.history['val_loss'], label='test')\npyplot.legend()\npyplot.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-06T13:36:08.930677Z","iopub.execute_input":"2022-10-06T13:36:08.931176Z","iopub.status.idle":"2022-10-06T13:37:05.405546Z","shell.execute_reply.started":"2022-10-06T13:36:08.931141Z","shell.execute_reply":"2022-10-06T13:37:05.404361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Carregando o melhor modelo**","metadata":{}},{"cell_type":"code","source":"from keras.models import load_model\nmodel=load_model('best_model.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:06:45.414067Z","iopub.execute_input":"2022-10-06T14:06:45.415132Z","iopub.status.idle":"2022-10-06T14:06:45.447977Z","shell.execute_reply.started":"2022-10-06T14:06:45.41509Z","shell.execute_reply":"2022-10-06T14:06:45.446368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defina a função que prevê o texto para o áudio fornecido:","metadata":{}},{"cell_type":"code","source":"def predict(audio):\n    prob=model.predict(audio.reshape(1,8000,1))\n    index=np.argmax(prob[0])\n    return classes[index]","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:08:54.666236Z","iopub.execute_input":"2022-10-06T14:08:54.66679Z","iopub.status.idle":"2022-10-06T14:08:54.673782Z","shell.execute_reply.started":"2022-10-06T14:08:54.666752Z","shell.execute_reply":"2022-10-06T14:08:54.672586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nindex=random.randint(0,len(x_val)-1)\nsamples=x_val[index].ravel()\nprint(\"Audio:\",classes[np.argmax(y_val[index])])\nipd.Audio(samples, rate=8000)","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:12:12.659747Z","iopub.execute_input":"2022-10-06T14:12:12.660198Z","iopub.status.idle":"2022-10-06T14:12:12.671061Z","shell.execute_reply.started":"2022-10-06T14:12:12.660161Z","shell.execute_reply":"2022-10-06T14:12:12.669796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Text:\",predict(samples))","metadata":{"execution":{"iopub.status.busy":"2022-10-06T14:12:16.879786Z","iopub.execute_input":"2022-10-06T14:12:16.880241Z","iopub.status.idle":"2022-10-06T14:12:16.952257Z","shell.execute_reply.started":"2022-10-06T14:12:16.880204Z","shell.execute_reply":"2022-10-06T14:12:16.950844Z"},"trusted":true},"execution_count":null,"outputs":[]}]}