{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<center><img src=\"https://raw.githubusercontent.com/dimitreOliveira/MachineLearning/master/Kaggle/Rainforest%20Connection%20Species%20Audio%20Detection/banner.png\" width=\"1000\"></center>\n<br>\n<center><h1>Rainforest - Audio classification Tensorflow starter</h1></center>\n<br>\n\n#### References:\n- [Simple audio recognition: Recognizing keywords](https://www.tensorflow.org/tutorials/audio/simple_audio)\n- [RFCX: train resnet50 with TPU](https://www.kaggle.com/yosshi999/rfcx-train-resnet50-with-tpu)\n- [Getting Started: Rainforest EDA with TFRecords](https://www.kaggle.com/jessemostipak/getting-started-rainforest-eda-with-tfrecords)","metadata":{"papermill":{"duration":0.033843,"end_time":"2020-12-15T16:30:31.108718","exception":false,"start_time":"2020-12-15T16:30:31.074875","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Workflow diagram\n```\n1. Load TFRecords\n2. Decode audio waveform\n3. Crop audio waveform\n4. Convert the waveform into spectrogram\n4. Resize spectrogram\n5. Turn grayscale spectrogram into RGB image\n6. Feed to the model\n```\n \n<center><img src=\"https://raw.githubusercontent.com/dimitreOliveira/MachineLearning/master/Kaggle/Rainforest%20Connection%20Species%20Audio%20Detection/Rainforest%20diagram.jpg\" width=\"1000\"></center>","metadata":{}},{"cell_type":"markdown","source":"## Dependencies","metadata":{"papermill":{"duration":0.039314,"end_time":"2020-12-15T16:30:31.307081","exception":false,"start_time":"2020-12-15T16:30:31.267767","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install --quiet efficientnet","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"papermill":{"duration":11.342642,"end_time":"2020-12-15T16:30:42.681143","exception":false,"start_time":"2020-12-15T16:30:31.338501","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, re, warnings, random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom IPython import display as Idisplay\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras import optimizers, losses, metrics, Model\nfrom tensorflow.keras.callbacks import EarlyStopping\nimport efficientnet.tfkeras as efn\nimport tensorflow_addons as tfa\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":8.53675,"end_time":"2020-12-15T16:30:51.250248","exception":false,"start_time":"2020-12-15T16:30:42.713498","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hardware configuration","metadata":{"papermill":{"duration":0.031658,"end_time":"2020-12-15T16:30:51.313854","exception":false,"start_time":"2020-12-15T16:30:51.282196","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_kg_hide-input":true,"papermill":{"duration":29.579927,"end_time":"2020-12-15T16:31:20.925583","exception":false,"start_time":"2020-12-15T16:30:51.345656","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model parameters","metadata":{"papermill":{"duration":0.031914,"end_time":"2020-12-15T16:31:20.990217","exception":false,"start_time":"2020-12-15T16:31:20.958303","status":"completed"},"tags":[]}},{"cell_type":"code","source":"BATCH_SIZE = 16 * REPLICAS\nLEARNING_RATE = 1e-3 * REPLICAS\nEPOCHS = 15\nHEIGHT = 224\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 24\nES_PATIENCE = 3\nTTA_STEPS = 6 # Do TTA if > 0 ","metadata":{"papermill":{"duration":0.044397,"end_time":"2020-12-15T16:31:21.067877","exception":false,"start_time":"2020-12-15T16:31:21.02348","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data\n\nWe have two different `.csv` files, `train_tp.csv` has the information of all true positive species labels and `train_fp.csv` has the information of all false positive species labels.","metadata":{"papermill":{"duration":0.034407,"end_time":"2020-12-15T16:31:21.135327","exception":false,"start_time":"2020-12-15T16:31:21.10092","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\ndatabase_base_path = '/kaggle/input/rfcx-species-audio-detection/'\ntrain_fp = pd.read_csv(f'{database_base_path}train_fp.csv')\ntrain_tp = pd.read_csv(f'{database_base_path}train_tp.csv')\n\nprint(f'Train false positive samples: {len(train_fp)}')\ndisplay(train_fp.head())\nprint(f'Train true positive samples: {len(train_tp)}')\ndisplay(train_tp.head())\n\nGCS_PATH = KaggleDatasets().get_gcs_path('rfcx-species-audio-detection')\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/train/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test/*.tfrec')\n\nNUM_TRAINING_SAMPLES = count_data_items(TRAINING_FILENAMES)\nNUM_TEST_SAMPLES = count_data_items(TEST_FILENAMES)\n\nprint(f'GCS: train samples: {NUM_TRAINING_SAMPLES}')\nprint(f'GCS: test samples: {NUM_TEST_SAMPLES}')","metadata":{"_kg_hide-input":true,"papermill":{"duration":2.368874,"end_time":"2020-12-15T16:31:23.536952","exception":false,"start_time":"2020-12-15T16:31:21.168078","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Auxiliary functions","metadata":{"papermill":{"duration":0.034562,"end_time":"2020-12-15T16:31:23.782284","exception":false,"start_time":"2020-12-15T16:31:23.747722","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Datasets utility functions\ndef decode_audio(audio_binary):\n    \"\"\"\n        Decode a 16-bit PCM WAV file to a float tensor.\n    \"\"\"\n    audio, _ = tf.audio.decode_wav(audio_binary)\n    return tf.squeeze(audio, axis=-1)\n    return audio\n\ndef string_split_semicolon(column):\n    split_labels_sc = tf.strings.split(column, sep=';')\n    return split_labels_sc\n\ndef string_split_comma(column):\n    split_labels_c = tf.strings.split(column, sep=',')\n    return split_labels_c\n\ndef get_label_info(label_info):\n    first_split = string_split_semicolon(label_info)\n    remove_quotes = tf.strings.regex_replace(first_split, '\"', \"\")\n    label_info = string_split_comma(remove_quotes)\n    return label_info\n\ndef get_spectrogram(waveform, padding=False, min_padding=48000):\n    \"\"\"\n        Transforms a 'waveform' into a 'spectrogram', adding padding if needed.\n    \"\"\"\n    waveform = tf.cast(waveform, tf.float32)\n    if padding:\n        # Padding for files with less than {min_padding} samples\n        zero_padding = tf.zeros([min_padding] - tf.shape(waveform), dtype=tf.float32)\n        # Concatenate audio with padding so that all audio clips will be of the same length\n        waveform = tf.concat([waveform, zero_padding], 0)\n    spectrogram = tf.signal.stft(waveform, frame_length=2048, frame_step=512, fft_length=2048)\n    spectrogram = tf.abs(spectrogram)\n    return spectrogram\n    \ndef get_spectrogram_tf(example):\n    \"\"\"\n        Transforms a 'waveform' tensor into a 'spectrogram'.\n        Applied to a Tensorflow dataset.\n    \"\"\"\n    audio = example['audio_wav']\n    spectrogram = get_spectrogram(audio)\n    spectrogram = tf.expand_dims(spectrogram, -1)\n    example['audio_wav'] = spectrogram\n    return example\n\ndef prepare_sample(example):\n    \"\"\"\n        1. Resize samples to the expected size.\n        2. Convert gray scales (1 channel) images to RGB (3 channels) format.\n    \"\"\"\n    sample = example['audio_wav']\n    sample = tf.image.resize(sample, [HEIGHT, WIDTH])\n    sample = tf.image.grayscale_to_rgb(sample)\n    example['audio_wav'] = sample\n    return example\n\ndef crop_audio(audio, tmin, tmax, crop_size=10, sample_rate=48000, max_size=60):\n    \"\"\"\n        Crops a 'waveform' file to have {crop_size} size given, {tmin}, {tmax}, {sample_rate} and {max_size}.\n    \"\"\"\n    label_size = tmax - tmin\n    \n    if label_size >= crop_size: # No padding needed\n        cut_min = tmin * sample_rate\n        cut_max = (tmin + crop_size) * sample_rate\n    else: # Needs padding\n        if tmin <= (max_size - crop_size): # Pad at the end\n            cut_min = tmin * sample_rate\n            cut_max = (tmin + crop_size) * sample_rate\n        else: # Pad at the beginning\n            cut_min = (tmin - crop_size) * sample_rate\n            cut_max = tmax * sample_rate\n    \n    # Casting tensors\n    cut_min = tf.cast(cut_min, tf.int32)\n    cut_max = tf.cast(cut_max, tf.int32)\n    cut_size = tf.cast((crop_size*sample_rate), tf.int32)\n    \n    audio = audio[cut_min:cut_max] # croping the audio\n    audio = audio[:cut_size] # making sure it has the max size\n    \n    audio = tf.reshape(audio, [cut_size]) # making sure it has the expected shape\n    return audio\n\ndef random_crop_audio(audio, crop_size=10, sample_rate=48000, max_size=60):\n    \"\"\"\n        Randomly crops a 'waveform' file to have {crop_size} size given, {sample_rate} and {max_size}.\n    \"\"\"\n    start = tf.random.uniform([], minval=0, \n                              maxval=(max_size - crop_size), \n                              dtype=tf.int32)\n    cut_min = start * sample_rate\n    cut_max = (start + crop_size) * sample_rate\n    \n    # Casting tensors\n    cut_min = tf.cast(cut_min, tf.int32)\n    cut_max = tf.cast(cut_max, tf.int32)\n    cut_size = tf.cast((crop_size*sample_rate), tf.int32)\n    \n    audio = audio[cut_min:cut_max] # croping the audio\n    audio = audio[:cut_size] # making sure it has the max size\n    \n    audio = tf.reshape(audio, [cut_size]) # making sure it has the expected shape\n    return audio\n\ndef read_tfrecord(example, labeled=True, inference=False):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode PCM WAV file.\n        3. Break down the information from 'label_info' into other features.\n        4. Crop the 'audio' waveform if needed.\n        5. Returns the features as a dictionary.\n    \"\"\"\n    TFREC_FORMAT = {\n        'audio_wav': tf.io.FixedLenFeature([], tf.string), \n        'recording_id': tf.io.FixedLenFeature([], tf.string), \n        'label_info': tf.io.FixedLenFeature([], tf.string, default_value='-1,-1,0,0,0,0,1'), \n    }\n        \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    audio = decode_audio(example['audio_wav'])\n    \n    # Break down 'label_info' into the data columns\n    label_info = get_label_info(example['label_info'])\n    species_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 0]), tf.int32)\n#     songtype_id = tf.strings.to_number(tf.gather_nd(label_info, [0, 1]), tf.int32)\n    tmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 2]))\n#     fmin = tf.strings.to_number(tf.gather_nd(label_info, [0, 3]))\n    tmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 4]))\n#     fmax = tf.strings.to_number(tf.gather_nd(label_info, [0, 5]))\n    is_tp = tf.strings.to_number(tf.gather_nd(label_info, [0, 6]), tf.int32)\n\n    if labeled:\n        audio = crop_audio(audio, tmin, tmax)\n    if inference:\n        audio = random_crop_audio(audio)\n        \n    features = {'audio_wav': audio, \n                'recording_id': example['recording_id'], \n                'species_id': species_id, \n                'is_tp': is_tp\n               }\n    return features\n\ndef load_dataset(filenames, labeled=True, ordered=False, inference=False):\n    \"\"\"\n        Load and parse the TFRecords.\n    \"\"\"\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n        dataset = tf.data.Dataset.list_files(filenames)\n        dataset = dataset.interleave(tf.data.TFRecordDataset, num_parallel_calls=AUTO)\n    else:\n        dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)    \n    dataset = dataset.map(lambda x: read_tfrecord(x, labeled=labeled, inference=inference), num_parallel_calls=AUTO)\n    return dataset\n\ndef conf_output(sample, labeled=True):\n    \"\"\"\n        Configure the output of the dataset.\n    \"\"\"\n    output = ({'input_audio': sample['audio_wav']}, sample['species_id'])\n    return output\n\ndef get_dataset(filenames, labeled=True, ordered=False, repeated=False, inference=False):\n    \"\"\"\n        1. Load TFRecord files, parse and generate features (waveform and meta-data).\n        2. Filter the dataset to contain only true positive samples.\n        3. Create 'spectrogram' from the 'waveform'.\n        4. Prepare image for the model.\n        5. Configure data to have the expected output format.\n        6. Apply Tensorflow data functions to optimize training.\n        \n        Returns a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = load_dataset(filenames, labeled=labeled, inference=inference)\n    \n    if labeled:\n        dataset = dataset.filter(_filtterTP)\n    \n    dataset = dataset.map(get_spectrogram_tf, num_parallel_calls=AUTO)\n    dataset = dataset.map(prepare_sample, num_parallel_calls=AUTO)\n    dataset = dataset.map(lambda x: conf_output(x, labeled=labeled), num_parallel_calls=AUTO)\n    \n    if not ordered:\n        dataset = dataset.shuffle(256)\n    if repeated:\n        dataset = dataset.repeat()\n        \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef _filtterTP(x):\n    return x['is_tp'] == 1","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.071773,"end_time":"2020-12-15T16:31:24.017413","exception":false,"start_time":"2020-12-15T16:31:23.94564","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualization utility functions\ndef plot_spectrogram(spectrogram, ax):\n    # Convert to frequencies to log scale and transpose so that the time is represented in the x-axis (columns).\n    log_spec = np.log(spectrogram.T)\n    height = log_spec.shape[0]\n    X = np.arange(spectrogram.shape[0])\n    Y = range(height)\n    ax.pcolormesh(X, Y, log_spec)\n    \ndef display_waveforms(ds, n_rows=3, n_cols=3, figsize=(20, 16)):\n    n = n_rows*n_cols\n    fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n    for i, sample in enumerate(ds.take(n)):\n        r = i // n_cols\n        c = i % n_cols\n        ax = axes[r][c]\n        ax.plot(sample['audio_wav'].numpy())\n        ax.set_yticks(np.arange(-1.2, 1.2, 0.2))\n        label = sample['species_id'].numpy()\n        recording_id = sample['recording_id'].numpy().decode()\n        ax.set_title(f'{recording_id} - {label}')\n    plt.show()\n    \ndef display_spectrograms(ds, n_rows=3, n_cols=3, figsize=(20, 16)):\n    n = n_rows*n_cols\n    fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n    for i, sample in enumerate(ds.take(n)):\n        r = i // n_cols\n        c = i % n_cols\n        ax = axes[r][c]\n        plot_spectrogram(np.squeeze(sample['audio_wav'].numpy()), ax)\n        label = sample['species_id'].numpy()\n        recording_id = sample['recording_id'].numpy().decode()\n        ax.set_title(f'{recording_id} - {label}')\n    plt.show()\n    \ndef inspect_preds(features, labels, preds, n_rows=3, n_cols=2, figsize=(20, 16)):\n    n = n_rows*n_cols\n    fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n    for i, (feature, label, pred) in enumerate(zip(features, labels, preds)):\n        r = i // n_cols\n        c = i % n_cols\n        ax = axes[r][c]\n        plot_spectrogram(np.squeeze(feature), ax)\n        if pred == label:\n            color = 'black'\n            title = f'{pred} [True]'\n        else:\n            color = 'red'\n            title = f'{pred} [False, should be {label}]'\n        ax.set_title(title, fontsize=14, color=color)\n    plt.show()\n        \ndef display_waveforms_audio_spectrogram(ds, n_samples=1, sample_rate=48000):\n    for sample in ds.take(n_samples):\n        waveform = sample['audio_wav']\n        label = sample['species_id'].numpy()\n        recording_id = sample['recording_id'].numpy().decode()\n        spectrogram = get_spectrogram(waveform)\n\n        print(f'Name: {recording_id}')\n        print(f'Label: {label}')\n        print(f'Waveform shape: {waveform.shape}')\n        print(f'Spectrogram shape: {spectrogram.shape}')\n        print(f'Audio playback')\n        Idisplay.display(Idisplay.Audio(waveform, rate=sample_rate))\n        \n        fig, axes = plt.subplots(2, figsize=(12, 8))\n        timescale = np.arange(waveform.shape[0])\n        axes[0].plot(timescale, waveform.numpy())\n        axes[0].set_title('Waveform')\n        axes[0].set_xlim([0, waveform.shape[0]])\n        plot_spectrogram(spectrogram.numpy(), axes[1])\n        axes[1].set_title('Spectrogram')\n        plt.show()\n        \ndef inspect_preds(features, labels, preds, n_rows=3, n_cols=2, figsize=(20, 16)):\n    n = n_rows*n_cols\n    fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n    for i, (feature, label, pred) in enumerate(zip(features, labels, preds)):\n        r = i // n_cols\n        c = i % n_cols\n        ax = axes[r][c]\n        \n        feature = tf.image.rgb_to_grayscale(feature).numpy()\n        plot_spectrogram(np.squeeze(feature), ax)\n        if pred == label:\n            color = 'black'\n            title = f'{pred} [True]'\n        else:\n            color = 'red'\n            title = f'{pred} [False, should be {label}]'\n        ax.set_title(title, fontsize=14, color=color)\n    plt.show()\n    \n# Model evaluation\ndef plot_metrics(history):\n    fig, axes = plt.subplots(2, 1, sharex='col', figsize=(20, 8))\n    axes = axes.flatten()\n    \n    axes[0].plot(history['loss'], label='Train loss')\n    axes[0].plot(history['val_loss'], label='Validation loss')\n    axes[0].legend(loc='best', fontsize=16)\n    axes[0].set_title('Loss')\n    axes[0].axvline(np.argmin(history['loss']), linestyle='dashed')\n    axes[0].axvline(np.argmin(history['val_loss']), linestyle='dashed', color='orange')\n    \n    axes[1].plot(history['sparse_categorical_accuracy'], label='Train accuracy')\n    axes[1].plot(history['val_sparse_categorical_accuracy'], label='Validation accuracy')\n    axes[1].legend(loc='best', fontsize=16)\n    axes[1].set_title('Accuracy')\n    axes[1].axvline(np.argmax(history['sparse_categorical_accuracy']), linestyle='dashed')\n    axes[1].axvline(np.argmax(history['val_sparse_categorical_accuracy']), linestyle='dashed', color='orange')\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.092055,"end_time":"2020-12-15T16:31:24.144293","exception":false,"start_time":"2020-12-15T16:31:24.052238","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading audio files\n\nThe audio file will initially be read as a binary file, which you'll want to convert into a numerical tensor.\n\nTo load an audio file, you will use [tf.audio.decode_wav](https://www.tensorflow.org/api_docs/python/tf/audio/decode_wav), which returns the WAV-encoded audio as a Tensor and the sample rate.\n\nA WAV file contains time series data with a set number of samples per second. Each sample represents the amplitude of the audio signal at that specific time. In a 16-bit system, the values range from -32768 to 32767. The sample rate for this dataset is 16kHz. Note that [tf.audio.decode_wav](https://www.tensorflow.org/api_docs/python/tf/audio/decode_wav) will normalize the values to the range [-1.0, 1.0].\n\n#### Let's examine a few audio waveforms with their corresponding labels.","metadata":{}},{"cell_type":"code","source":"train_waveform_ds = load_dataset(TRAINING_FILENAMES)\n\ndisplay_waveforms(train_waveform_ds)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectrogram\n\nYou'll convert the waveform into a spectrogram, which shows frequency changes over time and can be represented as a 2D image. This can be done by applying the short-time Fourier transform (STFT) to convert the audio into the time-frequency domain.\n\nA Fourier transform ([tf.signal.fft](https://www.tensorflow.org/api_docs/python/tf/signal/fft)) converts a signal to its component frequencies, but loses all time information. The STFT ([tf.signal.stft](https://www.tensorflow.org/api_docs/python/tf/signal/stft)) splits the signal into windows of time and runs a Fourier transform on each window, preserving some time information, and returning a 2D tensor that you can run standard convolutions on.\n\nSTFT produces an array of complex numbers representing magnitude and phase. However, you'll only need the magnitude for this tutorial, which can be derived by applying [tf.abs](https://www.tensorflow.org/api_docs/python/tf/math/abs) on the output of [tf.signal.stft](https://www.tensorflow.org/api_docs/python/tf/signal/stft).\n\nChoose `frame_length` and `frame_step` parameters such that the generated spectrogram \"image\" is almost square. For more information on STFT parameters choice, you can refer to [this video](https://www.coursera.org/lecture/audio-signal-processing/stft-2-tjEQe) on audio signal processing.\n\nYou also want the waveforms to have the same length, so that when you convert it to a spectrogram image, the results will have similar dimensions. This can be done by simply zero padding the audio clips that are shorter than one second.","metadata":{}},{"cell_type":"markdown","source":"Next, you will explore the data. Compare the waveform, the spectrogram and the actual audio of a few examples from the dataset.","metadata":{}},{"cell_type":"code","source":"display_waveforms_audio_spectrogram(train_waveform_ds, n_samples=3)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Examine the spectrogram \"images\" for different samples of the datasets.","metadata":{}},{"cell_type":"code","source":"train_spectrogram_ds = train_waveform_ds.map(get_spectrogram_tf, num_parallel_calls=AUTO)\n\ndisplay_spectrograms(train_spectrogram_ds)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.116466,"end_time":"2020-12-15T16:36:40.857129","exception":false,"start_time":"2020-12-15T16:36:40.740663","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def model_fn(input_shape, N_CLASSES):\n    inputs = L.Input(shape=input_shape, name='input_audio')\n    base_model = efn.EfficientNetB0(input_tensor=inputs, \n                                    include_top=False, \n                                    weights='noisy-student',\n                                    drop_connect_rate=0.4)\n\n    x = L.GlobalAveragePooling2D()(base_model.output)\n    x = L.Dropout(.5)(x)\n    output = L.Dense(N_CLASSES, activation='softmax', name='output')(x)\n    \n    model = Model(inputs=inputs, outputs=output)\n\n    return model","metadata":{"papermill":{"duration":0.132119,"end_time":"2020-12-15T16:36:41.105573","exception":false,"start_time":"2020-12-15T16:36:40.973454","status":"completed"},"tags":[],"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.118064,"end_time":"2020-12-15T16:36:41.348655","exception":false,"start_time":"2020-12-15T16:36:41.230591","status":"completed"},"tags":[]}},{"cell_type":"code","source":"oof_pred = []; oof_labels = []\nidxT, idxV = train_test_split(range(32), test_size=0.2, random_state=seed)\n\n# Create train and validation sets\nTRAIN_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/tfrecords/train/%.2i*.tfrec' % x for x in idxT])\nVALID_FILENAMES = tf.io.gfile.glob([GCS_PATH + '/tfrecords/train/%.2i*.tfrec' % x for x in idxV])\nnp.random.shuffle(TRAINING_FILENAMES)\nct_train = count_data_items(TRAIN_FILENAMES)\nct_valid = count_data_items(VALID_FILENAMES)\nsteps_per_epoch = 32 #(ct_train // BATCH_SIZE)\n\nprint(f'TRAIN: {idxT} | {ct_train} samples')\nprint(f'VALID: {idxV} | {ct_valid} samples')\n\n## MODEL\nwith strategy.scope():\n    model = model_fn((None, None, CHANNELS), N_CLASSES)\n\n    model.compile(optimizer=tfa.optimizers.RectifiedAdam(lr=LEARNING_RATE, \n                                                         min_lr=1e-8, \n                                                         total_steps=int(steps_per_epoch*EPOCHS), \n                                                         warmup_proportion=0.3), \n                  loss=losses.SparseCategoricalCrossentropy(), \n                  metrics=[metrics.SparseCategoricalAccuracy()])\n\nmodel_path = f'model.h5'\nes = EarlyStopping(monitor='val_sparse_categorical_accuracy', mode='max', \n                   patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n\n## TRAIN\nhistory = model.fit(x=get_dataset(TRAIN_FILENAMES, labeled=True, ordered=False, repeated=True), \n                    validation_data=get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False), \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS, \n                    callbacks=[es], \n                    verbose=2).history\n\n# Save last model weights\nmodel.save_weights(model_path)\n\n# OOF predictions\nds_valid = get_dataset(VALID_FILENAMES, labeled=True, ordered=True, repeated=False)\noof_labels.append([target.numpy() for x, target in iter(ds_valid.unbatch())])\nx_oof = ds_valid.map(lambda x, target: x)\noof_pred.append(np.argmax(model.predict(x_oof), axis=-1))\n\n## RESULTS\nprint(f\"#### Accuracy = {np.max(history['val_sparse_categorical_accuracy']):.3f}\")","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"papermill":{"duration":4883.276789,"end_time":"2020-12-15T17:58:04.742648","exception":false,"start_time":"2020-12-15T16:36:41.465859","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model loss graph","metadata":{"papermill":{"duration":0.157888,"end_time":"2020-12-15T17:58:05.058708","exception":false,"start_time":"2020-12-15T17:58:04.90082","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plot_metrics(history)","metadata":{"_kg_hide-input":true,"papermill":{"duration":3.358115,"end_time":"2020-12-15T17:58:08.575321","exception":false,"start_time":"2020-12-15T17:58:05.217206","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation\n\nNow we can evaluate the performance of the model, first, we can evaluate the usual metrics like, `accuracy`, `precision`, `recall`, and `f1-score`, `scikit-learn` provides the perfect function for this `classification_report`.\n\n## OOF metrics","metadata":{"papermill":{"duration":0.166729,"end_time":"2020-12-15T17:58:08.923208","exception":false,"start_time":"2020-12-15T17:58:08.756479","status":"completed"},"tags":[]}},{"cell_type":"code","source":"y_true = np.concatenate(oof_labels)\ny_pred = np.concatenate(oof_pred)\n\nprint(classification_report(y_true, y_pred))","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.272952,"end_time":"2020-12-15T17:58:09.713073","exception":false,"start_time":"2020-12-15T17:58:09.440121","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Confusion matrix\n\nLet's also take a look at the confusion matrix, this will give us an idea about what classes the model is mixing or having a hard time.","metadata":{"papermill":{"duration":0.170907,"end_time":"2020-12-15T17:58:10.058673","exception":false,"start_time":"2020-12-15T17:58:09.887766","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(20, 12))\ncfn_matrix = confusion_matrix(y_true, y_pred)\ncfn_matrix = (cfn_matrix.T / cfn_matrix.sum(axis=1)).T\ndf_cm = pd.DataFrame(cfn_matrix)\nax = sns.heatmap(df_cm, cmap='Blues', annot=True, fmt='.2f', linewidths=.5)\nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.759856,"end_time":"2020-12-15T17:58:11.0081","exception":false,"start_time":"2020-12-15T17:58:10.248244","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize predictions\n\nFinally, it is a good practice to always inspect some of the model's prediction by looking at the data, this can give an idea if the model is getting some predictions wrong because the data is really hard, or if it is because the model is actually bad.","metadata":{"papermill":{"duration":0.173347,"end_time":"2020-12-15T17:58:11.3631","exception":false,"start_time":"2020-12-15T17:58:11.189753","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES, ordered=True)\n\nfor features, label in train_dataset.take(1):\n    preds = np.argmax(model.predict(features['input_audio']), axis=-1)[:6]\n    labels = label.numpy()[:6]\n    batch_features = features['input_audio'].numpy()[:6]\n    \ninspect_preds(batch_features, labels, preds)","metadata":{"_kg_hide-input":true,"papermill":{"duration":70.88803,"end_time":"2020-12-15T17:59:22.424177","exception":false,"start_time":"2020-12-15T17:58:11.536147","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test set predictions","metadata":{}},{"cell_type":"code","source":"test_size = count_data_items(TEST_FILENAMES)\ntest_ds = get_dataset(TEST_FILENAMES, labeled=False, ordered=True, repeated=True, inference=True)\nct_steps = TTA_STEPS * ((test_size/BATCH_SIZE) + 1)\nx_test = test_ds.map(lambda features, recording_id: features['input_audio'])\ntest_preds = model.predict(x_test, steps=ct_steps, verbose=1)[:(test_size * TTA_STEPS)]\ntest_preds = np.mean(test_preds.reshape(test_size, TTA_STEPS, N_CLASSES, order='F'), axis=1)\n\nnames_test_ds = load_dataset(TEST_FILENAMES, labeled=False, ordered=True)\nnames = [features['recording_id'].numpy().decode('utf-8') for features in iter(names_test_ds)]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'recording_id': names})\nfor column in range(N_CLASSES):\n    submission[f's{column}'] = test_preds[:, column]\n\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head(10))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicted classes distribution","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(18, 8))\nax = sns.countplot(y=test_preds.argmax(axis=-1), palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}