{"metadata":{"accelerator":"GPU","colab":{"name":"micro_speech_with_lstm_op.ipynb","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9181064,"sourceType":"datasetVersion","datasetId":5549179}],"dockerImageVersionId":30747,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"##### Copyright 2020 The TensorFlow Authors.","metadata":{"id":"fluF3_oOgkWF"}},{"cell_type":"code","source":"#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.","metadata":{"cellView":"form","id":"AJs7HHFmg1M9","execution":{"iopub.status.busy":"2024-08-25T10:38:16.981840Z","iopub.execute_input":"2024-08-25T10:38:16.982694Z","iopub.status.idle":"2024-08-25T10:38:17.007828Z","shell.execute_reply.started":"2024-08-25T10:38:16.982659Z","shell.execute_reply":"2024-08-25T10:38:17.006770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tensorflow==2.8.0","metadata":{"execution":{"iopub.status.busy":"2024-08-25T10:38:17.009628Z","iopub.execute_input":"2024-08-25T10:38:17.009949Z","iopub.status.idle":"2024-08-25T10:39:20.889743Z","shell.execute_reply.started":"2024-08-25T10:38:17.009924Z","shell.execute_reply":"2024-08-25T10:39:20.888405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Simple audio recognition: Recognizing keywords","metadata":{"id":"jYysdyb-CaWM"}},{"cell_type":"markdown","source":"This tutorial will show you how to build a basic speech recognition network that recognizes ten different words. It's important to know that real speech and audio recognition systems are much more complex, but like MNIST for images, it should give you a basic understanding of the techniques involved. Once you've completed this tutorial, you'll have a model that tries to classify a one second audio clip as \"down\", \"go\", \"left\", \"no\", \"right\", \"stop\", \"up\" and \"yes\".","metadata":{"id":"SPfDNFlb66XF"}},{"cell_type":"markdown","source":"## Setup\n\nImport necessary modules and dependencies.","metadata":{"id":"Go9C3uLL8Izc"}},{"cell_type":"code","source":"import os\nimport pathlib\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nimport tensorflow as tf\nprint(tf.version.VERSION)\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import models\nfrom IPython import display\n\n# Set seed for experiment reproducibility\nseed = 42\ntf.random.set_seed(seed)\nnp.random.seed(seed)","metadata":{"id":"dzLKpmZICaWN","outputId":"9ba23229-9705-42df-8f7b-c4c321660a9b","execution":{"iopub.status.busy":"2024-08-25T10:39:20.891637Z","iopub.execute_input":"2024-08-25T10:39:20.892742Z","iopub.status.idle":"2024-08-25T10:39:26.382116Z","shell.execute_reply.started":"2024-08-25T10:39:20.892693Z","shell.execute_reply":"2024-08-25T10:39:26.381017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import the Speech Commands dataset\n\nYou'll write a script to download a portion of the [Speech Commands dataset](https://www.tensorflow.org/datasets/catalog/speech_commands). The original dataset consists of over 105,000 WAV audio files of people saying thirty different words. This data was collected by Google and released under a CC BY license.\n\nYou'll be using a portion of the dataset to save time with data loading. Extract the `mini_speech_commands.zip` and load it in using the `tf.data` API.","metadata":{"id":"yR0EdgrLCaWR"}},{"cell_type":"code","source":"data_dir = pathlib.Path('data/mini_speech_commands')\nif not data_dir.exists():\n  tf.keras.utils.get_file(\n      'mini_speech_commands.zip',\n      origin=\"http://storage.googleapis.com/download.tensorflow.org/data/mini_speech_commands.zip\",\n      extract=True,\n      cache_dir='.', cache_subdir='data')","metadata":{"id":"2-rayb7-3Y0I","execution":{"iopub.status.busy":"2024-08-25T10:39:26.385157Z","iopub.execute_input":"2024-08-25T10:39:26.386196Z","iopub.status.idle":"2024-08-25T10:39:41.297829Z","shell.execute_reply.started":"2024-08-25T10:39:26.386158Z","shell.execute_reply":"2024-08-25T10:39:41.296731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Moving wav files from command directories to unknown sub-directory (Factory Reset to reset data directory)\n","metadata":{"id":"h_72nZHA9UH9"}},{"cell_type":"code","source":"# comment out below line if \"unknown\" directory already exists\n!mkdir /kaggle/working/unknown\n# moves files from their specific commands directory to the \"unknown\" directory (replaces all files with an existing name therefore example set is smaller)\n!mv /kaggle/input/mini-speech/mini_speech_commands/down/* /kaggle/working/unknown\n!mv /kaggle/input/mini-speech/mini_speech_commands/go/* /kaggle/working/unknown\n!mv /kaggle/input/mini-speech/mini_speech_commands/left/* /kaggle/working/unknown\n!mv /kaggle/input/mini-speech/mini_speech_commands/right/* /kaggle/working/unknown\n!mv /kaggle/input/mini-speech/mini_speech_commands/stop/* /kaggle/working/unknown\n!mv /kaggle/input/mini-speech/mini_speech_commands/up/* /kaggle/working/unknown\n!ls /kaggle/working/unknown | wc -l","metadata":{"id":"vB4Jq4Qg35iZ","outputId":"85d90934-908c-48b2-85b7-8500ad302e04","execution":{"iopub.status.busy":"2024-08-25T10:39:41.299736Z","iopub.execute_input":"2024-08-25T10:39:41.300169Z","iopub.status.idle":"2024-08-25T10:40:24.017360Z","shell.execute_reply.started":"2024-08-25T10:39:41.300132Z","shell.execute_reply":"2024-08-25T10:40:24.016037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sets wanted commands for training (Available commands: Down, Go, Left, No, Right, Stop, Up, Yes, and Unknown for commands that are not to be tested","metadata":{"id":"BgvFq3uYiS5G"}},{"cell_type":"code","source":"commands = np.array(tf.io.gfile.listdir(str(data_dir)))\ncommands = [\"yes\",\"no\", \"unknown\"]\nprint('Commands:', commands)","metadata":{"id":"70IBxSKxA1N9","outputId":"a22025ee-8e85-4377-b223-b1d804fe5a25","execution":{"iopub.status.busy":"2024-08-25T10:40:24.019384Z","iopub.execute_input":"2024-08-25T10:40:24.019865Z","iopub.status.idle":"2024-08-25T10:40:24.026920Z","shell.execute_reply.started":"2024-08-25T10:40:24.019821Z","shell.execute_reply":"2024-08-25T10:40:24.025966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extract the audio files into a list and shuffle it.","metadata":{"id":"aMvdU9SY8WXN"}},{"cell_type":"code","source":"filenames = tf.io.gfile.glob(str(data_dir) + '/*/*')\nfilenames = tf.random.shuffle(filenames)\nnum_samples = len(filenames)\nprint('Number of total examples:', num_samples)\nprint('Number of examples per label:',\n      len(tf.io.gfile.listdir(str(data_dir/commands[0]))))\nprint('Example file tensor:', filenames[0])","metadata":{"id":"hlX685l1wD9k","outputId":"c1847369-0ba1-4ddc-f817-e55e08fdced2","execution":{"iopub.status.busy":"2024-08-25T10:40:24.028476Z","iopub.execute_input":"2024-08-25T10:40:24.029215Z","iopub.status.idle":"2024-08-25T10:40:24.101111Z","shell.execute_reply.started":"2024-08-25T10:40:24.029178Z","shell.execute_reply":"2024-08-25T10:40:24.100112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split the files into training, validation and test sets using a 80:10:10 ratio, respectively.","metadata":{"id":"9vK3ymy23MCP"}},{"cell_type":"code","source":"# Take 80% of total number examples for training set files\ntrain_files = filenames[:4249]\n# Take 10% of total number examples adding to 80% of total examples for validation set files\nval_files = filenames[4249: 4249 + 531]\n# Take -10% of total number examples for test set files\ntest_files = filenames[-531:]\n\nprint('Training set size', len(train_files))\nprint('Validation set size', len(val_files))\nprint('Test set size', len(test_files))","metadata":{"id":"Cv_wts-l3KgD","outputId":"8d1c6417-0157-4945-f0ef-864a87ce6679","execution":{"iopub.status.busy":"2024-08-25T10:40:24.102499Z","iopub.execute_input":"2024-08-25T10:40:24.102815Z","iopub.status.idle":"2024-08-25T10:40:24.111480Z","shell.execute_reply.started":"2024-08-25T10:40:24.102790Z","shell.execute_reply":"2024-08-25T10:40:24.110304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading audio files and their labels","metadata":{"id":"g2Cj9FyvfweD"}},{"cell_type":"markdown","source":"The audio file will initially be read as a binary file, which you'll want to convert into a numerical tensor.\n\nTo load an audio file, you will use [`tf.audio.decode_wav`](https://www.tensorflow.org/api_docs/python/tf/audio/decode_wav), which returns the WAV-encoded audio as a Tensor and the sample rate.\n\nA WAV file contains time series data with a set number of samples per second.\nEach sample represents the amplitude of the audio signal at that specific time. In a 16-bit system, like the files in `mini_speech_commands`, the values range from -32768 to 32767.\nThe sample rate for this dataset is 16kHz.\nNote that `tf.audio.decode_wav` will normalize the values to the range [-1.0, 1.0].","metadata":{"id":"j1zjcWteOcBy"}},{"cell_type":"code","source":"def decode_audio(audio_binary):\n  audio, _ = tf.audio.decode_wav(audio_binary)\n  return tf.squeeze(audio, axis=-1)","metadata":{"id":"9PjJ2iXYwftD","execution":{"iopub.status.busy":"2024-08-25T10:40:24.112711Z","iopub.execute_input":"2024-08-25T10:40:24.113018Z","iopub.status.idle":"2024-08-25T10:40:24.122727Z","shell.execute_reply.started":"2024-08-25T10:40:24.112993Z","shell.execute_reply":"2024-08-25T10:40:24.121627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The label for each WAV file is its parent directory.","metadata":{"id":"GPQseZElOjVN"}},{"cell_type":"code","source":"def get_label(file_path):\n  parts = tf.strings.split(file_path, os.path.sep)\n\n  # Note: You'll use indexing here instead of tuple unpacking to enable this\n  # to work in a TensorFlow graph.\n  return parts[-2]","metadata":{"id":"8VTtX1nr3YT-","execution":{"iopub.status.busy":"2024-08-25T10:40:24.127774Z","iopub.execute_input":"2024-08-25T10:40:24.128141Z","iopub.status.idle":"2024-08-25T10:40:24.134089Z","shell.execute_reply.started":"2024-08-25T10:40:24.128112Z","shell.execute_reply":"2024-08-25T10:40:24.132891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's define a method that will take in the filename of the WAV file and output a tuple containing the audio and labels for supervised training.","metadata":{"id":"E8Y9w_5MOsr-"}},{"cell_type":"code","source":"def get_waveform_and_label(file_path):\n  label = get_label(file_path)\n  audio_binary = tf.io.read_file(file_path)\n  waveform = decode_audio(audio_binary)\n  return waveform, label","metadata":{"id":"WdgUD5T93NyT","execution":{"iopub.status.busy":"2024-08-25T10:40:24.135366Z","iopub.execute_input":"2024-08-25T10:40:24.135694Z","iopub.status.idle":"2024-08-25T10:40:24.149732Z","shell.execute_reply.started":"2024-08-25T10:40:24.135663Z","shell.execute_reply":"2024-08-25T10:40:24.148548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You will now apply `process_path` to build your training set to extract the audio-label pairs and check the results. You'll build the validation and test sets using a similar procedure later on.","metadata":{"id":"nvN8W_dDjYjc"}},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\nfiles_ds = tf.data.Dataset.from_tensor_slices(train_files)\nwaveform_ds = files_ds.map(get_waveform_and_label, num_parallel_calls=AUTOTUNE)","metadata":{"id":"0SQl8yXl3kNP","execution":{"iopub.status.busy":"2024-08-25T10:40:24.151208Z","iopub.execute_input":"2024-08-25T10:40:24.151630Z","iopub.status.idle":"2024-08-25T10:40:24.351861Z","shell.execute_reply.started":"2024-08-25T10:40:24.151602Z","shell.execute_reply":"2024-08-25T10:40:24.350888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's examine a few audio waveforms with their corresponding labels.","metadata":{"id":"voxGEwvuh2L7"}},{"cell_type":"code","source":"rows = 3\ncols = 3\nn = rows*cols\nfig, axes = plt.subplots(rows, cols, figsize=(10, 12))\nfor i, (audio, label) in enumerate(waveform_ds.take(n)):\n  r = i // cols\n  c = i % cols\n  ax = axes[r][c]\n  ax.plot(audio.numpy())\n  ax.set_yticks(np.arange(-1.2, 1.2, 0.2))\n  label = label.numpy().decode('utf-8')\n  ax.set_title(label)\n\nplt.show()","metadata":{"id":"8yuX6Nqzf6wT","outputId":"8575133a-16c0-4d08-9857-ee4ecbfbe2fd","execution":{"iopub.status.busy":"2024-08-25T10:40:24.353381Z","iopub.execute_input":"2024-08-25T10:40:24.353826Z","iopub.status.idle":"2024-08-25T10:40:26.270994Z","shell.execute_reply.started":"2024-08-25T10:40:24.353772Z","shell.execute_reply":"2024-08-25T10:40:26.269871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spectrogram\n\nYou'll convert the waveform into a spectrogram, which shows frequency changes over time and can be represented as a 2D image. This can be done by applying the short-time Fourier transform (STFT) to convert the audio into the time-frequency domain.\n\nA Fourier transform ([`tf.signal.fft`](https://www.tensorflow.org/api_docs/python/tf/signal/fft)) converts a signal to its component frequencies, but loses all time information. The STFT ([`tf.signal.stft`](https://www.tensorflow.org/api_docs/python/tf/signal/stft)) splits the signal into windows of time and runs a Fourier transform on each window, preserving some time information, and returning a 2D tensor that you can run standard convolutions on.\n\nSTFT produces an array of complex numbers representing magnitude and phase. However, you'll only need the magnitude for this tutorial, which can be derived by applying `tf.abs` on the output of `tf.signal.stft`.\n\nChoose `frame_length` and `frame_step` parameters such that the generated spectrogram \"image\" is almost square. For more information on STFT parameters choice, you can refer to [this video](https://www.coursera.org/lecture/audio-signal-processing/stft-2-tjEQe) on audio signal processing.\n\nYou also want the waveforms to have the same length, so that when you convert it to a spectrogram image, the results will have similar dimensions. This can be done by simply zero padding the audio clips that are shorter than one second.\n","metadata":{"id":"EWXPphxm0B4m"}},{"cell_type":"code","source":"def get_spectrogram(waveform):\n  # Padding for files with less than 16000 samples\n  zero_padding = tf.zeros([16000] - tf.shape(waveform), dtype=tf.float32)\n\n  # Concatenate audio with padding so that all audio clips will be of the\n  # same length\n  waveform = tf.cast(waveform, tf.float32)\n  equal_length = tf.concat([waveform, zero_padding], 0)\n  spectrogram = tf.signal.stft(\n      equal_length, frame_length=480, frame_step=320, fft_length=512)\n\n  spectrogram = tf.abs(spectrogram)\n\n  return spectrogram","metadata":{"id":"_4CK75DHz_OR","execution":{"iopub.status.busy":"2024-08-25T10:40:26.272657Z","iopub.execute_input":"2024-08-25T10:40:26.273032Z","iopub.status.idle":"2024-08-25T10:40:26.279587Z","shell.execute_reply.started":"2024-08-25T10:40:26.273003Z","shell.execute_reply":"2024-08-25T10:40:26.278369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, you will explore the data. Compare the waveform, the spectrogram and the actual audio of one example from the dataset.","metadata":{"id":"5rdPiPYJphs2"}},{"cell_type":"code","source":"for waveform, label in waveform_ds.take(1):\n  label = label.numpy().decode('utf-8')\n  spectrogram = get_spectrogram(waveform)\n\nprint('Label:', label)\nprint('Waveform shape:', waveform.shape)\nprint('Spectrogram shape:', spectrogram.shape)\nprint('Audio playback')\ndisplay.display(display.Audio(waveform, rate=16000))","metadata":{"id":"4Mu6Y7Yz3C-V","outputId":"cdb225d1-54f2-4554-e518-a9f32865f728","execution":{"iopub.status.busy":"2024-08-25T10:40:26.281119Z","iopub.execute_input":"2024-08-25T10:40:26.281496Z","iopub.status.idle":"2024-08-25T10:40:26.349067Z","shell.execute_reply.started":"2024-08-25T10:40:26.281467Z","shell.execute_reply":"2024-08-25T10:40:26.348082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_spectrogram(spectrogram, ax):\n  # Convert to frequencies to log scale and transpose so that the time is\n  # represented in the x-axis (columns).\n  log_spec = np.log(spectrogram.T)\n  height = log_spec.shape[0]\n  width = log_spec.shape[1]\n  X = np.linspace(0, np.size(spectrogram), num=width, dtype=int)\n  Y = range(height)\n  ax.pcolormesh(X, Y, log_spec)\n\n\nfig, axes = plt.subplots(2, figsize=(12, 8))\ntimescale = np.arange(waveform.shape[0])\naxes[0].plot(timescale, waveform.numpy())\naxes[0].set_title('Waveform')\naxes[0].set_xlim([0, 16000])\nplot_spectrogram(spectrogram.numpy(), axes[1])\naxes[1].set_title('Spectrogram')\nplt.show()","metadata":{"id":"e62jzb36-Jog","outputId":"bc3a0ea2-38d7-496a-d1d5-5048f99eeed6","execution":{"iopub.status.busy":"2024-08-25T10:40:26.350566Z","iopub.execute_input":"2024-08-25T10:40:26.351291Z","iopub.status.idle":"2024-08-25T10:40:26.831083Z","shell.execute_reply.started":"2024-08-25T10:40:26.351252Z","shell.execute_reply":"2024-08-25T10:40:26.829982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now transform the waveform dataset to have spectrogram images and their corresponding labels as integer IDs.","metadata":{"id":"GyYXjW07jCHA"}},{"cell_type":"code","source":"def get_spectrogram_and_label_id(audio, label):\n  spectrogram = get_spectrogram(audio)\n  spectrogram = tf.expand_dims(spectrogram, -1)\n  label_id = tf.argmax(label == commands)\n  return spectrogram, label_id","metadata":{"id":"43IS2IouEV40","execution":{"iopub.status.busy":"2024-08-25T10:40:26.832656Z","iopub.execute_input":"2024-08-25T10:40:26.833141Z","iopub.status.idle":"2024-08-25T10:40:26.839237Z","shell.execute_reply.started":"2024-08-25T10:40:26.833107Z","shell.execute_reply":"2024-08-25T10:40:26.838168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spectrogram_ds = waveform_ds.map(\n    get_spectrogram_and_label_id, num_parallel_calls=AUTOTUNE)\n","metadata":{"id":"yEVb_oK0oBLQ","execution":{"iopub.status.busy":"2024-08-25T10:40:26.840432Z","iopub.execute_input":"2024-08-25T10:40:26.840736Z","iopub.status.idle":"2024-08-25T10:40:27.037309Z","shell.execute_reply.started":"2024-08-25T10:40:26.840712Z","shell.execute_reply":"2024-08-25T10:40:27.036229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Examine the spectrogram \"images\" for different samples of the dataset.","metadata":{"id":"6gQpAAgMnyDi"}},{"cell_type":"code","source":"rows = 3\ncols = 3\nn = rows*cols\nfig, axes = plt.subplots(rows, cols, figsize=(10, 10))\nfor i, (spectrogram, label_id) in enumerate(spectrogram_ds.take(n)):\n  r = i // cols\n  c = i % cols\n  ax = axes[r][c]\n  plot_spectrogram(np.squeeze(spectrogram.numpy()), ax)\n  ax.set_title(commands[label_id.numpy()])\n  ax.axis('off')\n\nplt.show()","metadata":{"id":"QUbHfTuon4iF","outputId":"f5057001-c343-43c3-ca64-cff3030d3380","execution":{"iopub.status.busy":"2024-08-25T10:40:27.039158Z","iopub.execute_input":"2024-08-25T10:40:27.039529Z","iopub.status.idle":"2024-08-25T10:40:27.910297Z","shell.execute_reply.started":"2024-08-25T10:40:27.039500Z","shell.execute_reply":"2024-08-25T10:40:27.909164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build and train the model\n\nNow you can build and train your model. But before you do that, you'll need to repeat the training set preprocessing on the validation and test sets.","metadata":{"id":"z5KdY8IF8rkt"}},{"cell_type":"code","source":"def preprocess_dataset(files):\n  files_ds = tf.data.Dataset.from_tensor_slices(files)\n  output_ds = files_ds.map(get_waveform_and_label, num_parallel_calls=AUTOTUNE)\n  output_ds = output_ds.map(\n      get_spectrogram_and_label_id,  num_parallel_calls=AUTOTUNE)\n  return output_ds","metadata":{"id":"10UI32QH_45b","execution":{"iopub.status.busy":"2024-08-25T10:40:27.911826Z","iopub.execute_input":"2024-08-25T10:40:27.912147Z","iopub.status.idle":"2024-08-25T10:40:27.917433Z","shell.execute_reply.started":"2024-08-25T10:40:27.912121Z","shell.execute_reply":"2024-08-25T10:40:27.916392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = spectrogram_ds\nval_ds = preprocess_dataset(val_files)\ntest_ds = preprocess_dataset(test_files)\nprint(val_ds)\nprint(test_ds)\n","metadata":{"id":"HNv4xwYkB2P6","outputId":"19e46c86-5f05-4bf2-ac06-1ea1e5f03dfd","execution":{"iopub.status.busy":"2024-08-25T10:40:27.918748Z","iopub.execute_input":"2024-08-25T10:40:27.919142Z","iopub.status.idle":"2024-08-25T10:40:28.166932Z","shell.execute_reply.started":"2024-08-25T10:40:27.919111Z","shell.execute_reply":"2024-08-25T10:40:28.165880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Batch the training and validation sets for model training.","metadata":{"id":"assnWo6SB3lR"}},{"cell_type":"code","source":"batch_size = 64\ntrain_ds = train_ds.batch(batch_size)\nval_ds = val_ds.batch(batch_size)","metadata":{"id":"UgY9WYzn61EX","execution":{"iopub.status.busy":"2024-08-25T10:40:28.168364Z","iopub.execute_input":"2024-08-25T10:40:28.168725Z","iopub.status.idle":"2024-08-25T10:40:28.175370Z","shell.execute_reply.started":"2024-08-25T10:40:28.168689Z","shell.execute_reply":"2024-08-25T10:40:28.174313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add dataset [`cache()`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#cache) and [`prefetch()`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#prefetch) operations to reduce read latency while training the model.","metadata":{"id":"GS1uIh6F_TN9"}},{"cell_type":"code","source":"train_ds = train_ds.cache().prefetch(AUTOTUNE)\nval_ds = val_ds.cache().prefetch(AUTOTUNE)","metadata":{"id":"fdZ6M-F5_QzY","execution":{"iopub.status.busy":"2024-08-25T10:40:28.176850Z","iopub.execute_input":"2024-08-25T10:40:28.177282Z","iopub.status.idle":"2024-08-25T10:40:28.187701Z","shell.execute_reply.started":"2024-08-25T10:40:28.177246Z","shell.execute_reply":"2024-08-25T10:40:28.186561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the model, you'll use a simple convolutional neural network (CNN), since you have transformed the audio files into spectrogram images.\nThe model also has the following additional preprocessing layers:\n- A [`Resizing`](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/Resizing) layer to downsample the input to enable the model to train faster.\n- A [`Normalization`](https://www.tensorflow.org/api_docs/python/tf/keras/layers/experimental/preprocessing/Normalization) layer to normalize each pixel in the image based on its mean and standard deviation.\n\nFor the `Normalization` layer, its `adapt` method would first need to be called on the training data in order to compute aggregate statistics (i.e. mean and standard deviation).","metadata":{"id":"rwHkKCQQb5oW"}},{"cell_type":"code","source":"from tensorflow.keras import models, layers\n\nfor spectrogram, _ in spectrogram_ds.take(1):\n    input_shape = spectrogram.shape\n    print('Input shape:', input_shape)\n    num_labels = len(commands)\n    print('num_labels:', num_labels)\n\n    model = models.Sequential([\n        layers.Input(shape=(49, 257), name='input'),\n        layers.Reshape(target_shape=(49, 257)),\n        layers.LSTM(80, return_sequences=True),  # Removed time_major argument\n        layers.Flatten(),\n        layers.Dense(3, activation='softmax', name='output')\n    ])\n    model.summary()\n","metadata":{"id":"ALYz7PFCHblP","outputId":"41f2133f-823d-471f-9172-ece3c0b6ea03","execution":{"iopub.status.busy":"2024-08-25T10:40:28.189093Z","iopub.execute_input":"2024-08-25T10:40:28.189631Z","iopub.status.idle":"2024-08-25T10:40:28.593765Z","shell.execute_reply.started":"2024-08-25T10:40:28.189593Z","shell.execute_reply":"2024-08-25T10:40:28.592407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","metadata":{"id":"wFjj7-EmsTD-","execution":{"iopub.status.busy":"2024-08-25T10:40:28.596201Z","iopub.execute_input":"2024-08-25T10:40:28.596695Z","iopub.status.idle":"2024-08-25T10:40:28.616335Z","shell.execute_reply.started":"2024-08-25T10:40:28.596654Z","shell.execute_reply":"2024-08-25T10:40:28.615104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check the training and validation loss curves to see how your model has improved during training.","metadata":{"id":"gjpCDeQ4mUfS"}},{"cell_type":"code","source":"EPOCHS = 100\nhistory = model.fit(\n    train_ds,\n    validation_data=val_ds,\n    epochs=EPOCHS,\n   # callbacks=tf.keras.callbacks.EarlyStopping(verbose=1, patience=2),\n)","metadata":{"id":"ttioPJVMcGtq","outputId":"2fcf46d7-5704-4547-e8f4-b28e93cf474b","execution":{"iopub.status.busy":"2024-08-25T10:40:28.617745Z","iopub.execute_input":"2024-08-25T10:40:28.618071Z","iopub.status.idle":"2024-08-25T10:46:48.408917Z","shell.execute_reply.started":"2024-08-25T10:40:28.618045Z","shell.execute_reply":"2024-08-25T10:46:48.407422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metrics = history.history\nplt.plot(history.epoch, metrics['loss'], metrics['val_loss'])\nplt.legend(['loss', 'val_loss'])\nplt.show()","metadata":{"id":"nzhipg3Gu2AY","outputId":"a006e1e9-da10-4617-e699-01c5b04763dc","execution":{"iopub.status.busy":"2024-08-25T10:46:48.411907Z","iopub.execute_input":"2024-08-25T10:46:48.413059Z","iopub.status.idle":"2024-08-25T10:46:48.622202Z","shell.execute_reply.started":"2024-08-25T10:46:48.413023Z","shell.execute_reply":"2024-08-25T10:46:48.621132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate test set performance\n\nLet's run the model on the test set and check performance.","metadata":{"id":"5ZTt3kO3mfm4"}},{"cell_type":"code","source":"test_audio = []\ntest_labels = []\n\nfor audio, label in test_ds:\n  test_audio.append(audio.numpy())\n  test_labels.append(label.numpy())\n\ntest_audio = np.array(test_audio)\ntest_labels = np.array(test_labels)","metadata":{"id":"biU2MwzyAo8o","execution":{"iopub.status.busy":"2024-08-25T10:46:48.629757Z","iopub.execute_input":"2024-08-25T10:46:48.630120Z","iopub.status.idle":"2024-08-25T10:46:49.298936Z","shell.execute_reply.started":"2024-08-25T10:46:48.630092Z","shell.execute_reply":"2024-08-25T10:46:49.298042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(model.predict(test_audio), axis=1)\ny_true = test_labels\n\ntest_acc = sum(y_pred == y_true) / len(y_true)\nprint(f'Test set accuracy: {test_acc:.0%}')","metadata":{"id":"ktUanr9mRZky","outputId":"475fcebb-64af-4c8e-e4af-784289106a5b","execution":{"iopub.status.busy":"2024-08-25T10:46:49.300222Z","iopub.execute_input":"2024-08-25T10:46:49.300571Z","iopub.status.idle":"2024-08-25T10:46:50.101118Z","shell.execute_reply.started":"2024-08-25T10:46:49.300542Z","shell.execute_reply":"2024-08-25T10:46:50.100043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Display a confusion matrix\n\nA confusion matrix is helpful to see how well the model did on each of the commands in the test set.","metadata":{"id":"en9Znt1NOabH"}},{"cell_type":"code","source":"confusion_mtx = tf.math.confusion_matrix(y_true, y_pred)\nplt.figure(figsize=(10, 8))\nsns.heatmap(confusion_mtx, xticklabels=commands, yticklabels=commands,\n            annot=True, fmt='g')\nplt.xlabel('Prediction')\nplt.ylabel('Label')\nplt.show()","metadata":{"id":"LvoSAOiXU3lL","outputId":"d3a37e34-55f8-4ca6-d826-ac8255e8f36c","execution":{"iopub.status.busy":"2024-08-25T10:46:50.102452Z","iopub.execute_input":"2024-08-25T10:46:50.102775Z","iopub.status.idle":"2024-08-25T10:46:50.386569Z","shell.execute_reply.started":"2024-08-25T10:46:50.102749Z","shell.execute_reply":"2024-08-25T10:46:50.385383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run inference on an audio file\n\nFinally, verify the model's prediction output using an input audio file of someone saying \"yes.\" How well does your model perform?","metadata":{"id":"mQGi_mzPcLvl"}},{"cell_type":"code","source":"import os\n\nsample_file = '/kaggle/input/mini-speech/mini_speech_commands/yes/0135f3f2_nohash_1.wav'\nif not os.path.exists(sample_file):\n    print(\"File not found:\", sample_file)\nelse:\n    sample_ds = preprocess_dataset([str(sample_file)])\n    for spectrogram, label in sample_ds.batch(1):\n        prediction = model(spectrogram)\n        print(len(commands))\n        print(tf.nn.softmax(prediction[0]))\n        print(tf.nn.softmax(prediction))\n        plt.bar(commands, tf.nn.softmax(prediction[0]))\n        plt.title(f'Predictions for \"{commands[label[0]]}\"')\n        plt.show()\n","metadata":{"id":"zRxauKMdhofU","outputId":"de931674-31e5-42af-c7f2-30420aad5dbe","execution":{"iopub.status.busy":"2024-08-25T10:46:50.388035Z","iopub.execute_input":"2024-08-25T10:46:50.388481Z","iopub.status.idle":"2024-08-25T10:46:50.834824Z","shell.execute_reply.started":"2024-08-25T10:46:50.388445Z","shell.execute_reply":"2024-08-25T10:46:50.833916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can see that your model very clearly recognized the audio command as \"yes.\"","metadata":{"id":"VgWICqdqQNaQ"}},{"cell_type":"markdown","source":"##Run TF inference on multiple audio files","metadata":{"id":"NNPrzNrcDqq8"}},{"cell_type":"code","source":"import glob\nimport pandas as pd\npd.set_option(\"display.precision\", 2)\n\ntxtfiles = []\nfor file in glob.glob(\"/kaggle/working/unknown/*.wav\"):\n    txtfiles.append(file)\n\nfor i in range(25):\n  print(txtfiles[i])\n  sample_ds = preprocess_dataset([str(txtfiles[i])])\n  for spectrogram, label in sample_ds.batch(1):\n    prediction = model(spectrogram)\n    plt.bar(commands, tf.nn.softmax(prediction[0]))\n    plt.title(f'Predictions for \"{commands[label[0]]}\"')\n    plt.show()\n","metadata":{"id":"GmyOg4kwDEVH","outputId":"b770a9ce-6797-42bc-8b2f-06523463c84a","execution":{"iopub.status.busy":"2024-08-25T10:46:50.836240Z","iopub.execute_input":"2024-08-25T10:46:50.836579Z","iopub.status.idle":"2024-08-25T10:47:02.693959Z","shell.execute_reply.started":"2024-08-25T10:46:50.836551Z","shell.execute_reply":"2024-08-25T10:47:02.692756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Generate TF Lite float32 model","metadata":{"id":"E8IcErTDyudr"}},{"cell_type":"code","source":"import tensorflow\nprint(tensorflow.__version__)\n\nrun_model = tf.function(lambda x: model(x))\n# This is important, let's fix the input size.\nBATCH_SIZE = 1\nSTEPS = 49\nINPUT_SIZE = 257\n\nconcrete_func = run_model.get_concrete_function(\n    tf.TensorSpec([BATCH_SIZE, STEPS, INPUT_SIZE], model.inputs[0].dtype))\n\n# model directory.\nMODEL_DIR = \"keras_lstm\"\nmodel.save(MODEL_DIR, save_format=\"tf\", signatures=concrete_func)\n\n#Float LSTM model\nconverter = tf.lite.TFLiteConverter.from_saved_model(MODEL_DIR)\ntflite_float_model = converter.convert()\nopen('/kaggle/working/keras_lstm/model_float.tflite', \"wb\").write(tflite_float_model)\n","metadata":{"id":"nlP5R7Y7ytYU","outputId":"7b8fee49-7b77-4bf5-fe17-aa593fafaa52","execution":{"iopub.status.busy":"2024-08-25T10:47:02.695749Z","iopub.execute_input":"2024-08-25T10:47:02.696362Z","iopub.status.idle":"2024-08-25T10:47:10.540917Z","shell.execute_reply.started":"2024-08-25T10:47:02.696285Z","shell.execute_reply":"2024-08-25T10:47:10.539847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Validate audio files using TFLite float32 model","metadata":{"id":"6lo_G30v0raq"}},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport glob\nimport pandas as pd\n\n# Load quantized TFLite model\ntflite_interpreter_float = tf.lite.Interpreter(model_path='/kaggle/working/keras_lstm/model_float.tflite')\n# Learn about its input and output details\ninput_details = tflite_interpreter_float.get_input_details()\noutput_details = tflite_interpreter_float.get_output_details()\ntflite_interpreter_float.allocate_tensors()\n\n# Ensure the correct float format for displaying DataFrames\npd.set_option('display.float_format', '{:.1f}'.format)\n\ntxtfiles = glob.glob(\"/kaggle/working/data/mini_speech_commands/no/*.wav\")\n\nfor i in range(25):\n    sample_ds = preprocess_dataset([str(txtfiles[i])])\n    print(txtfiles[i])\n    \n    # Run inference\n    for spectrogram, label in sample_ds.batch(1):\n        tflite_interpreter_float.set_tensor(input_details[0]['index'], np.array(spectrogram, dtype=np.float32).reshape(1,49, 257))\n        tflite_interpreter_float.invoke()\n        tflite_float_model_predictions = tflite_interpreter_float.get_tensor(output_details[0]['index'])\n        \n        # Convert prediction results to Pandas DataFrame, for better visualization\n        tflite_pred_dataframe = pd.DataFrame(tflite_float_model_predictions)\n        tflite_pred_dataframe.columns = commands\n        print(tflite_pred_dataframe)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T10:47:10.542474Z","iopub.execute_input":"2024-08-25T10:47:10.542820Z","iopub.status.idle":"2024-08-25T10:47:16.086119Z","shell.execute_reply.started":"2024-08-25T10:47:10.542792Z","shell.execute_reply":"2024-08-25T10:47:16.085020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load quantized TFLite model\n#  tflite_interpreter_float = tf.lite.Interpreter(model_path='/kaggle/working/keras_lstm/model_float.tflite')\n# # Learn about its input and output details\n# input_details = tflite_interpreter_float.get_input_details()\n# output_details = tflite_interpreter_float.get_output_details()\n# tflite_interpreter_float.allocate_tensors()\n\n# import glob\n# import pandas as pd\n# pd.set_option(\"display.precision\", 2)\n\n# txtfiles = []\n# for file in glob.glob(\"/kaggle/working/data/mini_speech_commands/no/*.wav\"):\n#     txtfiles.append(file)\n\n# for i in range(25):\n#   sample_ds = preprocess_dataset([str(txtfiles[i])])\n#   print(txtfiles[i])\n#   # Run inference\n#   for spectrogram, label in sample_ds.batch(1):\n#     tflite_interpreter_float.set_tensor(input_details[0]['index'],np.array(spectrogram, dtype=np.float32).reshape(1,49, 257) )\n#     tflite_interpreter_float.invoke()\n#     tflite_float_model_predictions = tflite_interpreter_float.get_tensor(output_details[0]['index'])\n#     # Convert prediction results to Pandas dataframe, for better visualization\n#     # Increase precision of presented data for better side-by-side comparison\n#     tflite_pred_dataframe = pd.DataFrame(tflite_float_model_predictions)\n#     tflite_pred_dataframe.columns = commands\n#     pd.set_option(\"precision\",1)\n#     print(tflite_pred_dataframe)\n","metadata":{"id":"t807WqXQ0qEJ","outputId":"16c5640e-e982-47ad-b25a-c414bc2e2780","execution":{"iopub.status.busy":"2024-08-25T10:47:16.088316Z","iopub.execute_input":"2024-08-25T10:47:16.088779Z","iopub.status.idle":"2024-08-25T10:47:17.051536Z","shell.execute_reply.started":"2024-08-25T10:47:16.088740Z","shell.execute_reply":"2024-08-25T10:47:17.050112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Generate Quantized int8 model\n","metadata":{"id":"VJANb9Uk-Xw-"}},{"cell_type":"code","source":"def representative_dataset_3():\n  for spectrogram, _ in spectrogram_ds.take(800):\n   # print('test')\n    flattened_data = np.array(spectrogram, dtype=np.float32).reshape(1,49, 257)\n    yield [flattened_data]\n\nconverter.representative_dataset = representative_dataset_3\nconverter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]\nconverter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]\nconverter.inference_input_type = tf.int8\nconverter.inference_output_type = tf.int8\nquantized_tflite_model = converter.convert()\nopen('/kaggle/working/keras_lstm/model_quantized_minispeech.tflite', \"wb\").write(quantized_tflite_model)\n","metadata":{"id":"nfMYKQgz-W_X","outputId":"28dc6ce9-24ab-44b6-bc79-4e65708b1d2c","execution":{"iopub.status.busy":"2024-08-25T10:48:38.476705Z","iopub.execute_input":"2024-08-25T10:48:38.477643Z","iopub.status.idle":"2024-08-25T10:48:41.723722Z","shell.execute_reply.started":"2024-08-25T10:48:38.477608Z","shell.execute_reply":"2024-08-25T10:48:41.722666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Validate Quantized int8 model","metadata":{"id":"HX4Ib7o4-kcH"}},{"cell_type":"code","source":"#test\n# Load quantized TFLite model\ntflite_interpreter_quant_int8 = tf.lite.Interpreter(model_path='/kaggle/working/keras_lstm/model_quantized_minispeech.tflite')\n# Learn about its input and output details\ninput_details = tflite_interpreter_quant_int8.get_input_details()\n\noutput_details = tflite_interpreter_quant_int8.get_output_details()\n\ntflite_interpreter_quant_int8.allocate_tensors()\n\nimport glob\nimport pandas as pd\npd.set_option(\"display.precision\", 2)\n\ntxtfiles = []\nfor file in glob.glob(\"/kaggle/working//data/mini_speech_commands/no/*.wav\"):\n    txtfiles.append(file)\n\n\nfor i in range(25):\n  print(txtfiles[i])\n  sample_ds = preprocess_dataset([str(txtfiles[i])])\n  # Run inference\n  for spectrogram, label in sample_ds.batch(1):\n    input_scale, input_zero_point = input_details[0][\"quantization\"]\n    print(input_scale)\n    print(input_zero_point)\n    spectrogram = np.array(spectrogram)\n    spectrogram = np.clip(np.floor(spectrogram / input_scale + input_zero_point), -128, 127) # for int8 validation\n    q_spectrogram = np.array(spectrogram, dtype=np.int8).reshape(1,49, 257)\n    tflite_interpreter_quant_int8.set_tensor(input_details[0]['index'], q_spectrogram )\n    tflite_interpreter_quant_int8.invoke()\n    tflite_q_model_predictions = tflite_interpreter_quant_int8.get_tensor(output_details[0]['index'])\n    print(tflite_q_model_predictions)\n    output_scale, output_zero_point = output_details[0][\"quantization\"]\n    tflite_model_predictions = (np.array(tflite_q_model_predictions, dtype=np.float32) - output_zero_point) * output_scale\n    tflite_pred_dataframe = pd.DataFrame(tflite_model_predictions)\n    tflite_pred_dataframe.columns = commands\n    print(tflite_pred_dataframe)","metadata":{"id":"cEwmoArnIcK9","outputId":"887b437e-0a79-4ff6-e422-0bf6d8e0d8aa","execution":{"iopub.status.busy":"2024-08-25T10:48:55.039486Z","iopub.execute_input":"2024-08-25T10:48:55.040312Z","iopub.status.idle":"2024-08-25T10:49:00.787275Z","shell.execute_reply.started":"2024-08-25T10:48:55.040277Z","shell.execute_reply":"2024-08-25T10:49:00.786149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Load quantized TFLite model\ntflite_interpreter_quant = tf.lite.Interpreter(model_path='/kaggle/working/keras_lstm/model_quantized_minispeech.tflite')\n# Learn about its input and output details\ninput_details = tflite_interpreter_quant.get_input_details()\n\noutput_details = tflite_interpreter_quant.get_output_details()\n\ntflite_interpreter_quant.allocate_tensors()\n\nimport glob\nimport pandas as pd\npd.set_option(\"display.precision\", 2)\n\ntxtfiles = []\nfor file in glob.glob(\"/kaggle/working/data/mini_speech_commands/no/*.wav\"):\n    txtfiles.append(file)\n\nfor i in range(25):\n  print(txtfiles[i])\n  sample_ds = preprocess_dataset([str(txtfiles[i])])\n  # Run inference\n  for spectrogram, label in sample_ds.batch(1):\n    spectrogram_t = np.array(spectrogram, dtype=np.uint8).reshape(1,49, 257)\n    spectrogram_t = np.array(spectrogram_t-128, dtype=np.int8)\n    tflite_interpreter_quant.set_tensor(input_details[0]['index'],spectrogram_t )\n    tflite_interpreter_quant.invoke()\n    tflite_q_model_predictions = tflite_interpreter_quant.get_tensor(output_details[0]['index'])\n    tflite_pred_dataframe = pd.DataFrame(tflite_q_model_predictions)\n    tflite_pred_dataframe.columns = commands\n    print(tflite_pred_dataframe)\n\n","metadata":{"id":"K0fLf2sx-jHx","outputId":"710765e4-05e8-4eec-98b2-36a693465370","execution":{"iopub.status.busy":"2024-08-25T10:49:35.169728Z","iopub.execute_input":"2024-08-25T10:49:35.170201Z","iopub.status.idle":"2024-08-25T10:49:40.864706Z","shell.execute_reply.started":"2024-08-25T10:49:35.170168Z","shell.execute_reply":"2024-08-25T10:49:40.863590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndef print_tflite_model_structure(model_path):\n    # Load the TFLite model\n    interpreter = tf.lite.Interpreter(model_path=model_path)\n    interpreter.allocate_tensors()\n\n    # Get tensor details\n    tensor_details = interpreter.get_tensor_details()\n\n    # Print input details\n    print(\"=== Model Inputs ===\")\n    input_details = interpreter.get_input_details()\n    for input_tensor in input_details:\n        print(f\"Name: {input_tensor['name']}, Shape: {input_tensor['shape']}, Type: {input_tensor['dtype']}\")\n\n    # Print output details\n    print(\"\\n=== Model Outputs ===\")\n    output_details = interpreter.get_output_details()\n    for output_tensor in output_details:\n        print(f\"Name: {output_tensor['name']}, Shape: {output_tensor['shape']}, Type: {output_tensor['dtype']}\")\n\n    # Print tensor details\n    print(\"\\n=== Model Tensor Details ===\")\n    for i, tensor in enumerate(tensor_details):\n        print(f\"Tensor {i}: Name: {tensor['name']}, Shape: {tensor['shape']}, Type: {tensor['dtype']}, Quantization: {tensor['quantization']}\")\n        \n# Path to your TFLite model\nmodel_path = '/kaggle/working/keras_lstm/model_quantized_minispeech.tflite'\n\n# Print the model structure\nprint_tflite_model_structure(model_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T10:49:50.447038Z","iopub.execute_input":"2024-08-25T10:49:50.447882Z","iopub.status.idle":"2024-08-25T10:49:50.461535Z","shell.execute_reply.started":"2024-08-25T10:49:50.447845Z","shell.execute_reply":"2024-08-25T10:49:50.460458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##Evaluate int8 model using floor and inputscale and zeropoint","metadata":{"id":"qg8U2AZkSfCH"}},{"cell_type":"markdown","source":"##Generate a TensorFlow Lite for MicroControllers Model\nConvert the TensorFlow Lite model into a C source file that can be loaded by TensorFlow Lite for Microcontrollers.","metadata":{"id":"tUToqbr-OqKp"}},{"cell_type":"code","source":"# Install xxd if it is not available\n!apt-get update && apt-get -qq install xxd\n# Convert to a C source file\n!xxd -i /content/keras_lstm/model_quantized_minispeech.tflite > /content/keras_lstm/model.cc\n","metadata":{"id":"_y90gcAtOs0-","outputId":"a422f82c-7a13-4d1e-8642-e8f7d63a114d","execution":{"iopub.status.busy":"2024-08-25T10:47:17.060584Z","iopub.status.idle":"2024-08-25T10:47:17.060946Z","shell.execute_reply.started":"2024-08-25T10:47:17.060775Z","shell.execute_reply":"2024-08-25T10:47:17.060789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\n\n# Load the quantized TFLite model\ntflite_interpreter = tf.lite.Interpreter(model_path='/kaggle/working/keras_lstm/model_quantized_minispeech.tflite')\ntflite_interpreter.allocate_tensors()\n\n# Get details of all tensors in the model\ntensor_details = tflite_interpreter.get_tensor_details()\n# tensor_details.detach().cpu().numpy()\nmytensor_shapes = {}\n# Attempt to get weights from valid tensor indices\ntry:\n    # Access known valid dense layer weights\n    dense_weights = tflite_interpreter.get_tensor(6)\n    print(f\"Weights for dense layer (MatMul1): {dense_weights}\")\n    \n    # Example for other tensors\n    for idx in range(len(tensor_details)):\n        try:\n            tensor = tflite_interpreter.get_tensor(idx)\n            mytensor_shapes[idx] = tensor.shape , tensor.dtype\n            if idx == 6:\n                print('9999999999999999999999999999')\n                tensor[2,3919] = 0\n            print(f\"Weights for tensor {idx}: {tensor}\")\n            print(tensor.shape)\n            print(\"********************************************\")\n\n        except ValueError as e:\n            print(f\"Error retrieving tensor {idx}: {e}\")\nexcept ValueError as e:\n    print(f\"Error retrieving dense layer weights: {e}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-08-25T12:57:23.497057Z","iopub.execute_input":"2024-08-25T12:57:23.497871Z","iopub.status.idle":"2024-08-25T12:57:23.516131Z","shell.execute_reply.started":"2024-08-25T12:57:23.497833Z","shell.execute_reply":"2024-08-25T12:57:23.514871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# +++++++**here**++++++++\n# ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\n\n# seed = 42\n# tf.random.set_seed(seed)\n# np.random.seed(seed)\n\ndef flatten_tensor(tensor):\n    return tensor.flatten()\n\ndef select_random_weights(flattened_array, ber):\n    # Randomly select indices\n    a =len(flattened_array)\n    print(a)\n    num_flips = int(len(flattened_array) * ber)\n    \n    indices = np.random.choice(len(flattened_array), num_flips, replace=False)\n    flattened_array[indices] = 0\n    return indices, flattened_array[indices]\n\ndef modify_random_weight_in_model(model_path, mytensor_shapes, ber):\n    interpreter = tf.lite.Interpreter(model_path=model_path)\n    interpreter.allocate_tensors()\n    tensor_details = interpreter.get_tensor_details()\n\n    # Initialize a list to store flattened tensors\n    flattened_tensors = []\n    \n    # Iterate over each tensor index defined in mytensor_shapes\n    for tensor_index, shape in mytensor_shapes.items():\n        tensor = interpreter.get_tensor(tensor_index)\n        flattened_tensor = flatten_tensor(tensor)\n        flattened_tensors.append(flattened_tensor)\n    \n    # Concatenate all flattened tensors into a single array\n    all_flattened_values = np.concatenate(flattened_tensors) if flattened_tensors else np.array([])\n    num_flips = int(len(all_flattened_values) * ber)\n\n    # Ensure there are enough values to select\n    if len(all_flattened_values) < num_flips:\n        raise ValueError(f\"Not enough weights to select. Available: {len(all_flattened_values)}, Requested: {num_flips}\")\n    \n    # Select random weights\n    indices, random_weights = select_random_weights(all_flattened_values, ber)\n    # Print the selected indices and corresponding weights\n    print(f\"Selected indices: {indices}\")\n    print(f\"Selected weights: {random_weights}\")\n    \n    \n    #covert array to tensor\n    \n#     reshaped = tf.reshape(all_flattened_values , [len(all_flattened_values)])\n#     print(reshaped)\n    \n\nmodel_path = '/kaggle/working/keras_lstm/model_quantized_minispeech.tflite'\n\nmytensor_shapes = {\n    0: ((1, 49, 257), np.int8),\n    1: ((2,), np.int32),\n    2: ((80,), np.int32),\n    3: ((80,), np.int32),\n    4: ((80,), np.int32),\n    5: ((80,), np.int32),\n    6: ((3, 3920), np.int8),\n    7: ((3,), np.int32),\n    8: ((80, 257), np.int8),\n    9: ((80, 257), np.int8),\n    10: ((80, 257), np.int8),\n    11: ((80, 257), np.int8),\n    12: ((80, 80), np.int8),\n    13: ((80, 80), np.int8),\n    14: ((80, 80), np.int8),\n    15: ((80, 80), np.int8),\n    16: ((1, 80), np.int8),\n    17: ((1, 80), np.int16),\n    23: ((1, 49, 80), np.int8),\n    24: ((1, 3920), np.int8),\n    25: ((1, 3), np.int8),\n    26: ((1, 3), np.int8),\n    27: ((1, 80), np.int16),\n    28: ((1, 80), np.int16),\n    29: ((1, 80), np.int16),\n    30: ((1, 80), np.int16),\n    31: ((1, 80), np.int8),\n    32: ((1, 80), np.int32)\n}\nmodify_random_weight_in_model(model_path, mytensor_shapes, ber=0.0001)","metadata":{"execution":{"iopub.status.busy":"2024-08-25T12:57:47.882521Z","iopub.execute_input":"2024-08-25T12:57:47.883433Z","iopub.status.idle":"2024-08-25T12:57:47.905851Z","shell.execute_reply.started":"2024-08-25T12:57:47.883394Z","shell.execute_reply":"2024-08-25T12:57:47.904669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**HERE++++++++++++++++++++++++++++++++***","metadata":{}}]}