{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Training pipeline with tensorflow dataset \n\nThe goal of this notebook is to process the audio files using Tensorflow Dataset API.\n\nThe model performance is not the purpose rather to show how to deal with a lot of data without overflowing the notebook's RAM. \n\nImplementation of articles on Medium by [David Schwertfeger](https://towardsdatascience.com/@davidschwertfeger?source=post_page-----b3133474c3c1----------------------)\n\nhttps://towardsdatascience.com/how-to-easily-process-audio-on-your-gpu-with-tensorflow-2d9d91360f06\n\nhttps://towardsdatascience.com/how-to-build-efficient-audio-data-pipelines-with-tensorflow-2-0-b3133474c3c1\n\n**Update version 11** \n\nUsing the resampled datasets from https://www.kaggle.com/c/birdsong-recognition/discussion/164197 \nThis allows to use `tf.audio.decode_wav` instead of `py_function` and is a lot lot faster !\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport librosa\nimport librosa.display\nimport pathlib\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom pydub import AudioSegment\n\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_CLASSES = 264 # Allows to reduce the number of classes to train (max = 264)\nSAMPLE_RATE = 32000 # Audio sample rate\nMAX_DURATION = 5 # Clip duration in seconds \nFFT_SIZE = 1024 # Fourier Transform size \nHOP_SIZE = 512 # Number of samples between each successive FFT window\nN_MEL_BINS = 128 \nN_SPECTROGRAM_BINS = (FFT_SIZE // 2) + 1\nF_MIN = 20# Min frequency cutoff\nF_MAX = SAMPLE_RATE / 2  # Max Frequency cutoff\nBATCH_SIZE = 64  # Training Batch size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/birdsong-recognition/train.csv\", parse_dates=['date'])\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Build a filepath column from the train.csv metadata ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"input_paths = {'a':'../input/birdsong-resampled-train-audio-00',\n               'b': '../input/birdsong-resampled-train-audio-00',\n               'c': '../input/birdsong-resampled-train-audio-01',\n               'e': '../input/birdsong-resampled-train-audio-01',\n               'f': '../input/birdsong-resampled-train-audio-01',\n               'g': '../input/birdsong-resampled-train-audio-02',\n               'h': '../input/birdsong-resampled-train-audio-02',\n               'i': '../input/birdsong-resampled-train-audio-02',\n               'j': '../input/birdsong-resampled-train-audio-02',\n               'k': '../input/birdsong-resampled-train-audio-02',\n               'l': '../input/birdsong-resampled-train-audio-02',\n               'm': '../input/birdsong-resampled-train-audio-02',\n               'n': '../input/birdsong-resampled-train-audio-03',\n               'o': '../input/birdsong-resampled-train-audio-03',\n               'p': '../input/birdsong-resampled-train-audio-03',\n               'q': '../input/birdsong-resampled-train-audio-03',\n               'r': '../input/birdsong-resampled-train-audio-03',\n               's': '../input/birdsong-resampled-train-audio-04',\n               't': '../input/birdsong-resampled-train-audio-04',\n               'u': '../input/birdsong-resampled-train-audio-04',\n               'v': '../input/birdsong-resampled-train-audio-04',\n               'w': '../input/birdsong-resampled-train-audio-04',\n               'x': '../input/birdsong-resampled-train-audio-04',\n               'y': '../input/birdsong-resampled-train-audio-04'          \n        }\n\n\n\n\ntrain['filepath'] = train[\"ebird_code\"].str[0].map(input_paths) + '/' + train[\"ebird_code\"] + '/' + train[\"filename\"]\ntrain['filepath'] = train['filepath'].str.replace('.mp3', '.wav')\ntrain['ebird_code'].value_counts()[:N_CLASSES]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Encode the label for training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.dropna(subset=['filepath'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\ntrain['label'] = le.fit_transform(train['ebird_code'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## We can then use librosa on the dataset to build the MEL Spectrogram","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Build the tensorflow Dataset and decode wav files with tf.audio\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\n\ndef get_dataset(df, label_column, filepath_column):\n    file_path_ds = tf.data.Dataset.from_tensor_slices(df[filepath_column].astype(bytes))\n    label_ds = tf.data.Dataset.from_tensor_slices(df[label_column])\n    return tf.data.Dataset.zip((file_path_ds, label_ds))\n\n\ndef load_audio(file_path, label):\n    audio = tf.io.read_file(file_path)\n    audio, sample_rate = tf.audio.decode_wav(audio,\n                                             desired_channels=1, \n                                             desired_samples = SAMPLE_RATE * 60  # take first 60 seconds (no offset possible ..)\n                                           )\n    #audio = tf.transpose(audio)\n    audio = tf.image.random_crop(audio, size=[SAMPLE_RATE * MAX_DURATION, 1]) # Random crop to 5 seconds\n    return audio, label\n\n\n\ndef prepare_for_training(ds, shuffle_buffer_size=512, batch_size=64):\n    # Randomly shuffle (file_path, label) dataset\n    ds = ds.shuffle(buffer_size=shuffle_buffer_size)\n    # Load and decode audio from file paths\n    ds = ds.map(load_audio, num_parallel_calls=AUTOTUNE)\n    # Repeat dataset forever\n    ds = ds.repeat()\n    # Prepare batches\n    ds = ds.batch(batch_size)\n    # Prefetch\n    ds = ds.prefetch(buffer_size=AUTOTUNE)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_audio(train.loc[0, 'filepath'], 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain, val = train_test_split(train, stratify=train['label'], test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = get_dataset(train, 'label', 'filepath')\nval_ds = get_dataset(val, 'label', 'filepath')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Custom preprocessing Layer for MELSpectrogram\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class LogMelSpectrogram(tf.keras.layers.Layer):\n    \"\"\"Compute log-magnitude mel-scaled spectrograms.\"\"\"\n\n    def __init__(self, sample_rate, fft_size, hop_size, n_mels,\n                 f_min=0.0, f_max=None, **kwargs):\n        super(LogMelSpectrogram, self).__init__(**kwargs)\n        self.sample_rate = sample_rate\n        self.fft_size = fft_size\n        self.hop_size = hop_size\n        self.n_mels = n_mels\n        self.f_min = f_min\n        self.f_max = f_max if f_max else sample_rate / 2\n        self.mel_filterbank = tf.signal.linear_to_mel_weight_matrix(\n            num_mel_bins=self.n_mels,\n            num_spectrogram_bins=fft_size // 2 + 1,\n            sample_rate=self.sample_rate,\n            lower_edge_hertz=self.f_min,\n            upper_edge_hertz=self.f_max)\n\n    def build(self, input_shape):\n        self.non_trainable_weights.append(self.mel_filterbank)\n        super(LogMelSpectrogram, self).build(input_shape)\n\n    def call(self, waveforms):\n        \"\"\"Forward pass.\n\n        Parameters\n        ----------\n        waveforms : tf.Tensor, shape = (None, n_samples)\n            A Batch of mono waveforms.\n\n        Returns\n        -------\n        log_mel_spectrograms : (tf.Tensor), shape = (None, time, freq, ch)\n            The corresponding batch of log-mel-spectrograms\n        \"\"\"\n        def _tf_log10(x):\n            numerator = tf.math.log(x)\n            denominator = tf.math.log(tf.constant(10, dtype=numerator.dtype))\n            return numerator / denominator\n\n        def power_to_db(magnitude, amin=1e-16, top_db=80.0):\n            \"\"\"\n            https://librosa.github.io/librosa/generated/librosa.core.power_to_db.html\n            \"\"\"\n            ref_value = tf.reduce_max(magnitude)\n            log_spec = 10.0 * _tf_log10(tf.maximum(amin, magnitude))\n            log_spec -= 10.0 * _tf_log10(tf.maximum(amin, ref_value))\n            log_spec = tf.maximum(log_spec, tf.reduce_max(log_spec) - top_db)\n\n            return log_spec\n\n        spectrograms = tf.signal.stft(waveforms,\n                                      frame_length=self.fft_size,\n                                      frame_step=self.hop_size,\n                                      pad_end=False)\n\n        magnitude_spectrograms = tf.abs(spectrograms)\n\n        mel_spectrograms = tf.matmul(tf.square(magnitude_spectrograms),\n                                     self.mel_filterbank)\n\n        log_mel_spectrograms = power_to_db(mel_spectrograms)\n\n        # add channel dimension\n        log_mel_spectrograms = tf.expand_dims(log_mel_spectrograms, 3)\n        return log_mel_spectrograms\n\n    def get_config(self):\n        config = {\n            'fft_size': self.fft_size,\n            'hop_size': self.hop_size,\n            'n_mels': self.n_mels,\n            'sample_rate': self.sample_rate,\n            'f_min': self.f_min,\n            'f_max': self.f_max,\n        }\n        config.update(super(LogMelSpectrogram, self).get_config())\n\n        return config","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#resnet = tf.keras.applications.ResNet50(\n#    include_top=False, weights='imagenet', input_shape=(311, 128, 3),\n#    pooling='avg'\n#)\n\nimport tensorflow_hub as hub\n\nfeature_extractor_url = \"https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/4\"\nfeature_extractor_layer = hub.KerasLayer(feature_extractor_url,\n                                         input_shape=(311, 128, 3))\n\nfeature_extractor_layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## This allows us to build a model that does the LogSpectogram ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import (BatchNormalization, Conv2D, Dense,\n                                     Dropout, Flatten, Input, MaxPool2D)\nfrom tensorflow.keras.models import Model\n\ndef ConvModel(n_classes, sample_rate=SAMPLE_RATE, duration=MAX_DURATION,\n              fft_size=FFT_SIZE, hop_size=HOP_SIZE, n_mels=N_MEL_BINS, fmin=F_MIN, fmax=F_MAX):\n    n_samples = sample_rate * duration\n    input_shape = (n_samples,)\n\n    x = Input(shape=input_shape, name='input', dtype='float32')    \n    y = LogMelSpectrogram(sample_rate, fft_size, hop_size, n_mels, fmin, fmax)(x)\n    y = BatchNormalization(axis=2)(y)\n\n\n    y = Conv2D(3, (3,3), padding='same')(y)  \n    y = BatchNormalization()(y)\n\n    y = feature_extractor_layer(y, training=False)\n\n    y = Dense(1024, activation='relu')(y)\n    y = Dropout(0.1)(y)\n    y = Dense(1024, activation='relu')(y)\n    y = Dropout(0.1)(y)\n    \n    y = Dense(n_classes, activation='softmax')(y)\n\n    return Model(inputs=x, outputs=y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.optimizers import SGD, schedules\n\nn_classes = train['label'].max() + 1\nmodel = ConvModel(n_classes)\n\nlr_schedule = schedules.ExponentialDecay(\n    initial_learning_rate=0.05, decay_steps=1000, decay_rate=0.96, staircase=False\n)\nsgd = SGD(learning_rate=lr_schedule, momentum=0.85)\nmodel.compile(optimizer=sgd,\n              loss='sparse_categorical_crossentropy', \n              metrics=['sparse_categorical_accuracy'])\n\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_ds = prepare_for_training(train_ds)\nvalid_ds = prepare_for_training(val_ds)\n\nsteps_per_epoch = len(train)//BATCH_SIZE\nsteps_per_epoch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_filepath = '/checkpoint'\nmodel_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n    filepath=checkpoint_filepath,\n    save_weights_only=True,\n    monitor='val_acc',\n    mode='max',\n    save_best_only=True)\n\nearly_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n\nmodel.fit(training_ds, \n          epochs=50, \n          steps_per_epoch=steps_per_epoch, \n          validation_data=valid_ds, \n          validation_steps=2, \n         callbacks=[model_checkpoint_callback, early_stop])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Stuck to a local minima of around 5.5 and 0.004 , notably 0.004 = 1/260 which you be the approximate accuracy if model predicts all in one class ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(\"resnet_model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for pred, label in valid_ds.take(2):\n    pred = pred.numpy()\n    label = label.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict = model.predict(pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[pred.argmax() for pred in predict]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.where(predict[0]>0.008)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.where(predict[1]>0.008)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}