{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":2315253,"sourceType":"datasetVersion","datasetId":1389941},{"sourceId":5821823,"sourceType":"datasetVersion","datasetId":3112969}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pre-processing | Data loading | Data Augmentation | CutMix/MixUp in PyTorch\nThis notebook goes over the preprocessing of audio samples with Mel-spectrograms in pytorch, the applications of common audio and spectrogram augmentations with **audiomentations**, **torch-audiomentations** and **torchaudio**, and the use of CutMix and MixUp within the dataloader using **torchvision**. Since this data loading step is heavy if performed on CPU and can be a bottleneck for training, we will see how we can perform these computations on GPU for a more than 2x speed-up, at the cost of some GPU memory.","metadata":{}},{"cell_type":"code","source":"# Install necessary packages\n!pip install audiomentations # CPU (numpy) augmentations\n!pip install torch-audiomentations # GPU (torch) augmentations","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:46:29.568221Z","iopub.execute_input":"2024-05-19T19:46:29.56917Z","iopub.status.idle":"2024-05-19T19:47:00.954514Z","shell.execute_reply.started":"2024-05-19T19:46:29.569123Z","shell.execute_reply":"2024-05-19T19:47:00.953486Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\nimport torchaudio\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport IPython.display as ipd\nfrom datetime import datetime\nimport time\nimport librosa\nimport ast\nplt.style.use('ggplot')\n\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import default_collate\nfrom torchvision.transforms import v2\n\nimport audiomentations\nimport torch_audiomentations","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-19T19:47:00.956553Z","iopub.execute_input":"2024-05-19T19:47:00.956887Z","iopub.status.idle":"2024-05-19T19:47:08.039162Z","shell.execute_reply.started":"2024-05-19T19:47:00.956857Z","shell.execute_reply":"2024-05-19T19:47:08.03829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utilities for displaying audio\ndef visualize_audio(waveform, sr, original=None):\n    if isinstance(waveform, torch.Tensor):\n        waveform = waveform.numpy()\n    if len(waveform.shape) > 1:\n        waveform = waveform[0]\n    ipd.display(ipd.Audio(waveform, rate=sr))\n\n    # Create subplots for audio visualizations\n    fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(14, 4))\n\n    # Waveform\n    if original is not None:\n        if isinstance(original, torch.Tensor):\n            original = original.numpy()\n        if len(original.shape) > 1:\n            original = original[0]\n        axs[0].plot(original, alpha=0.4, label='Original')\n    axs[0].plot(waveform, alpha=0.6, label='Augmented')\n    axs[0].set_title('Waveform')\n    if original is not None:\n        axs[0].legend()\n\n    # Mel Spectrogram\n    mel_spectrogram = librosa.feature.melspectrogram(y=waveform, sr=sr)\n    mel_spectrogram = librosa.power_to_db(mel_spectrogram, ref=np.max)\n    librosa.display.specshow(mel_spectrogram, x_axis='time', y_axis='mel', sr=sr, ax=axs[1])\n    axs[1].set_title('Mel Spectrogram')\n    \n    \ndef visualize_spec(spec):\n    plt.figure(figsize=(7,4))\n    plt.imshow(spec[0], origin='lower', cmap='magma')\n    plt.grid(visible=False)\n    plt.title('Mel Spectrogram')\n    plt.xlabel('Time step')\n    plt.ylabel('Mel bin')\n    plt.show()\n    \n    \ndef visualize_spec_batch(spec_batch):\n    plt.figure(figsize=(14,8))\n    for k in range(spec_batch.shape[0]):\n        plt.subplot(4,4,k+1)\n        plt.imshow(spec_batch[k,0].detach().cpu().numpy(), origin='lower', cmap='magma')\n        plt.grid(visible=False)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:08.040352Z","iopub.execute_input":"2024-05-19T19:47:08.040829Z","iopub.status.idle":"2024-05-19T19:47:08.054698Z","shell.execute_reply.started":"2024-05-19T19:47:08.040801Z","shell.execute_reply":"2024-05-19T19:47:08.053663Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data\n### Config\n\nConfig class for all parameters used in the notebook. STFT parameters are the same as the [https://www.kaggle.com/code/awsaf49/birdclef24-kerascv-starter-train](Keras starter notebook). Audio files are truncated to ```duration``` seconds, either randomly truncated or keeping only the beginning of the file depending on ```start_idx```. The parameters `n_mels` and `target_length` control the shape of the spectrograms (respectively the number of frequency bins and time bins). ","metadata":{}},{"cell_type":"code","source":"class Config:\n    start_idx = 'first'\n    duration = 10\n    sample_rate = 32000\n    target_length = 384\n    n_mels = 128\n    n_fft = 2028\n    window = 2028\n    audio_len = duration*sample_rate\n    hop_length = audio_len // (target_length-1)\n    fmin = 20\n    fmax = 16000\n    top_db = 80\n\n    n_classes = 182\n    batch_size = 16\n    model_name = 'efficientnet_v2_s'\n    \n    dataset_mean = [-16.8828]\n    dataset_std = [12.4019]\n\n    data_aug = True        \n    cutmix_mixup = False    \n    loss = 'bce'    # ('crossentropy', 'bce')\n    secondary_labels_weight = 0.3   \n\n    base_dir = '/kaggle/input/birdclef-2024'\n    short_noises = '/kaggle/input/birdclef-2023-additional/esc50/use_label'\n    background_noises = ['/kaggle/input/birdclef2021-background-noise/aicrowd2020_noise_30sec/noise_30sec',\n                         '/kaggle/input/birdclef2021-background-noise/ff1010bird_nocall/nocall',\n                         #'/kaggle/input/birdclef2021-background-noise/train_soundscapes/nocall'\n                        ]","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:08.057207Z","iopub.execute_input":"2024-05-19T19:47:08.057535Z","iopub.status.idle":"2024-05-19T19:47:08.07075Z","shell.execute_reply.started":"2024-05-19T19:47:08.05751Z","shell.execute_reply":"2024-05-19T19:47:08.069985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Files\nThe following reads the training files, stored in ```metadata['filepath']```. Training / vaidation split are ignored for this notebook.","metadata":{}},{"cell_type":"code","source":"train_dir = Config.base_dir + '/train_audio/'\nclass_names = sorted(os.listdir(train_dir))\nn_classes = len(class_names)\nclass_labels = list(range(n_classes))\nlabel2name = dict(zip(class_labels, class_names))\nname2label = {v:k for k,v in label2name.items()}\n\ndef get_label_from_name(name):\n    if name not in name2label.keys():\n        return None\n    return name2label[name]\n\nmetadata = pd.read_csv(Config.base_dir + '/train_metadata.csv')\nmetadata['filepath'] = train_dir + metadata.filename\nmetadata['target'] = metadata.primary_label.map(name2label)\nmetadata['secondary_targets'] = metadata.secondary_labels.map(lambda x: [get_label_from_name(name) for name in ast.literal_eval(x)])","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:08.071917Z","iopub.execute_input":"2024-05-19T19:47:08.07322Z","iopub.status.idle":"2024-05-19T19:47:08.489399Z","shell.execute_reply.started":"2024-05-19T19:47:08.073196Z","shell.execute_reply":"2024-05-19T19:47:08.488434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading audio files\nThe first step is reading the audio files. We use `torchaudio` for this, you can use `librosa` or others if you wish. All audio files are truncated (or padded) to the same duration, here 10s. We obtain torch tensors with 1 channel and 320000 time steps. Audio augmentations on CPU expect numpy arrays anyway.","metadata":{}},{"cell_type":"code","source":"# Truncated or pad the audio sample to the desired duration\ndef trunc_or_pad(waveform, audio_len, start_idx='random'):\n    sig_len = waveform.shape[-1]\n    diff_len = abs(sig_len - audio_len)\n\n    if (sig_len > audio_len):\n        # Truncate the signal to the given length\n        if start_idx == 'random':\n            start_idx = np.random.randint(0, diff_len)\n        else:\n            start_idx = 0\n        waveform = waveform[:, start_idx:start_idx + audio_len]\n    \n    elif (sig_len < audio_len):\n        # Length of padding to add at the beginning and end of the signal\n        pad1 = np.random.randint(0, diff_len)\n        pad2 = diff_len - pad1\n        if isinstance(waveform, torch.Tensor):\n            waveform = nn.functional.pad(waveform, pad=(pad1, pad2), mode='constant', value=0)\n        else:\n            waveform = np.pad(waveform, ((0, 0), (pad1, pad2)), mode='constant', constant_values=0)\n    \n    return waveform\n\nfile = metadata[\"filepath\"][0]\nwaveform, sr = torchaudio.load(file)\nwaveform = trunc_or_pad(waveform, Config.audio_len)\nwaveform.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:08.490688Z","iopub.execute_input":"2024-05-19T19:47:08.491005Z","iopub.status.idle":"2024-05-19T19:47:08.626775Z","shell.execute_reply.started":"2024-05-19T19:47:08.490979Z","shell.execute_reply":"2024-05-19T19:47:08.625859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_audio(waveform, sr)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:08.62799Z","iopub.execute_input":"2024-05-19T19:47:08.62829Z","iopub.status.idle":"2024-05-19T19:47:19.146857Z","shell.execute_reply.started":"2024-05-19T19:47:08.628266Z","shell.execute_reply":"2024-05-19T19:47:19.145945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Audio Augmentations on CPU with Audiomentations\n[Audiomentations](https://iver56.github.io/audiomentations/) is a python library for audio signal augmentations, similar to Albumentations for images. It implements many data augmentations for audio signal in the time domain. The audiomentations transforms expect numpy arrays (and not torch tensors) with the sample rate, and run on CPU. All transforms support application with random probability (controlled with the `p` parameter), and it implements the useful `Compose` and `OneOf` transforms. All available transforms are in the [documentation](https://iver56.github.io/audiomentations/), and can be visualized [here](https://phrasenmaeher-audio-transformat-visualize-transformation-5s1n4t.streamlit.app/). First we can view some useful augmentations in our case:","metadata":{}},{"cell_type":"markdown","source":"### Time Shift\nShift the samples forwards or backwards (randomly chosen), by default with rollover.","metadata":{}},{"cell_type":"code","source":"# Time shift\naug = audiomentations.Shift(min_shift=-0.5, max_shift=0.5, p=1)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:19.14808Z","iopub.execute_input":"2024-05-19T19:47:19.148516Z","iopub.status.idle":"2024-05-19T19:47:22.845602Z","shell.execute_reply.started":"2024-05-19T19:47:19.148488Z","shell.execute_reply":"2024-05-19T19:47:22.844662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pitch Shift\nPitch shift the sound up or down without changing the tempo (scales frequencies without rescaling the time steps).","metadata":{}},{"cell_type":"code","source":"# Pitch shift\naug = audiomentations.PitchShift(min_semitones=-2.5, max_semitones=2.5, p=1)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:22.846771Z","iopub.execute_input":"2024-05-19T19:47:22.847082Z","iopub.status.idle":"2024-05-19T19:47:25.462904Z","shell.execute_reply.started":"2024-05-19T19:47:22.847056Z","shell.execute_reply":"2024-05-19T19:47:25.461918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Equalizer\nAdjust the volume of different frequency bands.","metadata":{}},{"cell_type":"code","source":"# EQ\naug = audiomentations.SevenBandParametricEQ(min_gain_db=-12., max_gain_db=12., p=1)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:25.466415Z","iopub.execute_input":"2024-05-19T19:47:25.4668Z","iopub.status.idle":"2024-05-19T19:47:27.123889Z","shell.execute_reply.started":"2024-05-19T19:47:25.466762Z","shell.execute_reply":"2024-05-19T19:47:27.122947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Low pass filter\nCuts off high frequencies (higher than randomly picked cutoff frequency)","metadata":{}},{"cell_type":"code","source":"# Low pass filter\naug = audiomentations.LowPassFilter(min_cutoff_freq=750., max_cutoff_freq=7500., min_rolloff=12, max_rolloff=24, p=1)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:27.125435Z","iopub.execute_input":"2024-05-19T19:47:27.125737Z","iopub.status.idle":"2024-05-19T19:47:28.724297Z","shell.execute_reply.started":"2024-05-19T19:47:27.125705Z","shell.execute_reply":"2024-05-19T19:47:28.723312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Air absorbtion\nSimulates attenuation of high frequencies due to air absorption. High frequencies are attenuated quicker than low frequencies with the distance traveled, this transform simulates that. It does not simulate the global attenuation of the gain with the traveled distance.","metadata":{}},{"cell_type":"code","source":"# Attenuation due to distance traveled\naug = audiomentations.AirAbsorption(min_temperature=10, max_temperature=20, min_humidity=30, max_humidity=90,\n                                  min_distance=10, max_distance=100, p=1.)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:28.725764Z","iopub.execute_input":"2024-05-19T19:47:28.726439Z","iopub.status.idle":"2024-05-19T19:47:30.911823Z","shell.execute_reply.started":"2024-05-19T19:47:28.726401Z","shell.execute_reply":"2024-05-19T19:47:30.910892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gain \nMultiply the audio by a random amplitude factor to reduce or increase the volume. Helps the model to be somewhat invariant to the gain (volume) of the signal. ```GainTransition``` is a gradual transformation (fade in/fade out).","metadata":{}},{"cell_type":"code","source":"# Gain augmentations\naug = audiomentations.OneOf((\n    audiomentations.Gain(min_gain_db=-6., max_gain_db=6., p=1),\n    audiomentations.GainTransition(min_gain_db=-12., max_gain_db=3., p=1))\n)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:30.912864Z","iopub.execute_input":"2024-05-19T19:47:30.913206Z","iopub.status.idle":"2024-05-19T19:47:32.788372Z","shell.execute_reply.started":"2024-05-19T19:47:30.913181Z","shell.execute_reply":"2024-05-19T19:47:32.787456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gaussian noise\nAdd Gaussian noise (white noise). With ```AddGaussianSNR``` the noise amplitude is chosen relatively to the amplitude of the given signal, to obtain the ramdomly chosen SNR (Signal to Noise Ration), while in ```AddGaussianNoise``` the amplitude of the noise is independent from the input signal.","metadata":{}},{"cell_type":"code","source":"# Gaussian Noise\naug = audiomentations.OneOf((\n    audiomentations.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=1.),\n    audiomentations.AddGaussianSNR(min_snr_db=5., max_snr_db=40., p=1.))\n)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:32.789824Z","iopub.execute_input":"2024-05-19T19:47:32.790124Z","iopub.status.idle":"2024-05-19T19:47:34.374286Z","shell.execute_reply.started":"2024-05-19T19:47:32.790099Z","shell.execute_reply":"2024-05-19T19:47:34.373353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pink noise (and other colors)\nYou can find details about all colors of noise [here](https://en.wikipedia.org/wiki/Colors_of_noise), and how to obtain them in the [audiomentations documentation](https://iver56.github.io/audiomentations/waveform_transforms/add_color_noise/). Pink noise is commonly used as an augmentation.","metadata":{}},{"cell_type":"code","source":"# Pink Noise\naug = audiomentations.AddColorNoise(min_snr_db=5., max_snr_db=40., min_f_decay=-3.01, max_f_decay=-3.01, p=1.)\nwaveform_aug = aug(waveform.numpy(), sr)\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:34.375719Z","iopub.execute_input":"2024-05-19T19:47:34.376192Z","iopub.status.idle":"2024-05-19T19:47:35.954083Z","shell.execute_reply.started":"2024-05-19T19:47:34.376158Z","shell.execute_reply":"2024-05-19T19:47:35.953122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Background and Short Noises\nMix in another sound with the input signal. Background noises are mixed for the full duration of the input, with an amplitude relative to the input. Short noises are added only to a few seconds of the input. The audio files are from the follow datasets: [background](https://www.kaggle.com/datasets/christofhenkel/birdclef2021-background-noise?select=aicrowd2020_noise_30sec) and [short](https://www.kaggle.com/datasets/atsunorifujita/birdclef-2023-additional?select=train_datasets).\n\nRain and frog audio files from the ESC-50 dataset, background noises are samples without bird calls from previous competitions.","metadata":{}},{"cell_type":"code","source":"# Background noises from other files\naug = audiomentations.Compose([\n    audiomentations.AddShortNoises(sounds_path=Config.short_noises, min_snr_db=3., max_snr_db=30., \n                               noise_rms='relative_to_whole_input',\n                               min_time_between_sounds=2., max_time_between_sounds=8., \n                               noise_transform=audiomentations.PolarityInversion(), p=1.),\n    audiomentations.AddBackgroundNoise(sounds_path=Config.background_noises, min_snr_db=3., max_snr_db=30., \n                                   noise_transform=audiomentations.PolarityInversion(), p=1.)\n])\n\nwaveform_aug = aug(waveform.squeeze().numpy(), sr)[None, :] # These transforms want monochannel audio\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:35.955502Z","iopub.execute_input":"2024-05-19T19:47:35.955972Z","iopub.status.idle":"2024-05-19T19:47:44.467452Z","shell.execute_reply.started":"2024-05-19T19:47:35.955919Z","shell.execute_reply":"2024-05-19T19:47:44.466418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Radomly compose augmentations\nWe define the augmentations for the audio signal in the following:","metadata":{}},{"cell_type":"code","source":"# Combining random audio transforms\nwaveform_transforms = audiomentations.Compose([\n    audiomentations.Shift(min_shift=-0.5, max_shift=0.5, p=0.5),\n    audiomentations.SevenBandParametricEQ(min_gain_db=-12., max_gain_db=12., p=0.5),\n    audiomentations.AirAbsorption(min_temperature=10, max_temperature=20, min_humidity=30, max_humidity=90,\n                                  min_distance=10, max_distance=100, p=1.), \n\n    audiomentations.OneOf([\n        audiomentations.Gain(min_gain_db=-6., max_gain_db=6., p=1),\n        audiomentations.GainTransition(min_gain_db=-12., max_gain_db=3., p=1)\n    ], p=1.),\n\n    audiomentations.OneOf([\n        audiomentations.AddGaussianNoise(min_amplitude=0.001, max_amplitude=0.015, p=1.),\n        audiomentations.AddGaussianSNR(min_snr_db=5., max_snr_db=40., p=1.),\n        audiomentations.AddColorNoise(min_snr_db=5., max_snr_db=40., min_f_decay=-3.01, max_f_decay=-3.01, p=1.)\n    ], p=1.),\n\n    audiomentations.AddShortNoises(sounds_path=Config.short_noises, min_snr_db=3., max_snr_db=30., \n                               noise_rms='relative_to_whole_input',\n                               min_time_between_sounds=2., max_time_between_sounds=8., \n                               noise_transform=audiomentations.PolarityInversion(), p=0.5),\n    audiomentations.AddBackgroundNoise(sounds_path=Config.background_noises, min_snr_db=3., max_snr_db=30., \n                                   noise_transform=audiomentations.PolarityInversion(), p=0.5),\n                                   \n    audiomentations.LowPassFilter(min_cutoff_freq=750., max_cutoff_freq=7500., min_rolloff=12, max_rolloff=24, p=0.8),\n    audiomentations.PitchShift(min_semitones=-2.5, max_semitones=2.5, p=0.3)\n])\n\nwaveform_aug = waveform_transforms(waveform.squeeze().numpy(), sr)[None, :]\nvisualize_audio(waveform_aug, sr, original=waveform)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:44.469131Z","iopub.execute_input":"2024-05-19T19:47:44.469669Z","iopub.status.idle":"2024-05-19T19:47:47.333064Z","shell.execute_reply.started":"2024-05-19T19:47:44.469632Z","shell.execute_reply":"2024-05-19T19:47:47.331987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Spectrogram augmentations\nThe Mel-spectrogram is computed with torchaudio, in the dB scale, and then normalized using the statistics of the whole dataset. This is done to have the same transformation for all samples, and is the common procedure for natural images with ImageNet statistics (which are of course not adequate here). The mean and standard devation can be found in the Config class, and can be computed with the following code if using more samples from other datasets for example:\n```python\n# Compute dataset mean\nbase_mean = 0\nbase_std = 0\n\nfor k in tqdm(range(len(train_dataset))):\n    spec, _ = train_dataset[k]\n    base_mean += spec.mean()\n    base_std += spec.std()\n\nbase_mean = base_mean/len(train_dataset)\nbase_std = base_std/len(train_dataset)\n```","metadata":{}},{"cell_type":"code","source":"# Compute spectrogram\nto_mel_spectrogramn = nn.Sequential(\n    torchaudio.transforms.MelSpectrogram(Config.sample_rate, n_fft=Config.n_fft, win_length=Config.window,  \n                                         hop_length=Config.hop_length, n_mels=Config.n_mels, \n                                         f_min=Config.fmin, f_max=Config.fmax),\n    torchaudio.transforms.AmplitudeToDB(top_db=Config.top_db),\n    v2.Normalize(mean=Config.dataset_mean, std=Config.dataset_std)\n)\n\nspec = to_mel_spectrogramn(waveform)\nspec.shape\nvisualize_spec(spec)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:47.334388Z","iopub.execute_input":"2024-05-19T19:47:47.334766Z","iopub.status.idle":"2024-05-19T19:47:47.789679Z","shell.execute_reply.started":"2024-05-19T19:47:47.334732Z","shell.execute_reply":"2024-05-19T19:47:47.788788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Time and Frequency masking\nCommon augmentations for spectrograms are time and frequency masking, which respectively mask vertical and horizontal bands of the spectrograms. Keep in mind that most augmentations usually used with images make no sense for spectrograms, as they do not have the same invariances and symmetries of natural images. \n\nThe Transforms are cutom implementations based on torchaudio transforms to add randomness and multiple bands. The maximum number of masked bands is controled by ```n_masks```, the maximum width of the bands by ```max_mask_pct``` and the probability of each mask in ```n_masks``` by ```prob```. Masked bands are set to either zero or the mean of the spectrogram.","metadata":{"execution":{"iopub.status.busy":"2024-05-19T16:36:52.267436Z","iopub.execute_input":"2024-05-19T16:36:52.26783Z","iopub.status.idle":"2024-05-19T16:37:55.063367Z","shell.execute_reply.started":"2024-05-19T16:36:52.267797Z","shell.execute_reply":"2024-05-19T16:37:55.061235Z"}}},{"cell_type":"code","source":"# Spetrogram transforms\nclass FrequencyMaskingAug(torchaudio.transforms.FrequencyMasking):\n    def __init__(self, prob, max_mask_pct, n_mels, n_masks, mask_mode='mean'):\n        self.prob = prob\n        self.freq_mask_param = max_mask_pct * n_mels\n        self.n_masks = n_masks\n        self.mask_mode = mask_mode\n        super(FrequencyMaskingAug, self).__init__(self.freq_mask_param)\n    def forward(self, specgram):\n        if self.mask_mode == 'mean':\n            mask_value = specgram.mean()\n        else:\n            mask_value = 0\n        \n        for _ in range(self.n_masks):\n            # for batched inputs\n            if len(specgram.shape) > 3:\n                for k in range(specgram.size(0)):\n                    if np.random.random() < self.prob:\n                        if self.mask_mode == 'mean':\n                            mask_value = specgram[k].mean()\n                        else:\n                            mask_value = 0\n                        specgram[k] = super().forward(specgram[k], mask_value)\n            else:\n                if np.random.random() < self.prob:\n                    specgram = super().forward(specgram, mask_value)\n\n        return specgram\n    \nclass TimeMaskingAug(torchaudio.transforms.TimeMasking):\n    def __init__(self, prob, max_mask_pct, n_steps, n_masks, mask_mode='mean'):\n        self.prob = prob\n        self.time_mask_param = max_mask_pct * n_steps\n        self.n_masks = n_masks\n        self.mask_mode = mask_mode\n        super(TimeMaskingAug, self).__init__(self.time_mask_param)\n    def forward(self, specgram):\n        if self.mask_mode == 'mean':\n            mask_value = specgram.mean()\n        else:\n            mask_value = 0\n        \n        for _ in range(self.n_masks):\n            # for batched inputs\n            if len(specgram.shape) > 3:\n                for k in range(specgram.size(0)):\n                    if np.random.random() < self.prob:\n                        if self.mask_mode == 'mean':\n                            mask_value = specgram[k].mean()\n                        else:\n                            mask_value = 0\n                        specgram[k] = super().forward(specgram[k], mask_value)\n            else:\n                if np.random.random() < self.prob:\n                    specgram = super().forward(specgram, mask_value)\n\n        return specgram\n    \n\nspec_transforms = nn.Sequential(\n    FrequencyMaskingAug(0.3, 0.1, Config.n_mels, n_masks=3, mask_mode='mean'),\n    TimeMaskingAug(0.3, 0.1, Config.target_length, n_masks=3, mask_mode='mean'),\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:47.79115Z","iopub.execute_input":"2024-05-19T19:47:47.791746Z","iopub.status.idle":"2024-05-19T19:47:47.808502Z","shell.execute_reply.started":"2024-05-19T19:47:47.791711Z","shell.execute_reply":"2024-05-19T19:47:47.807538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spec_aug = spec_transforms(spec)\nvisualize_spec(spec_aug)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:47.809853Z","iopub.execute_input":"2024-05-19T19:47:47.810245Z","iopub.status.idle":"2024-05-19T19:47:48.140637Z","shell.execute_reply.started":"2024-05-19T19:47:47.81022Z","shell.execute_reply":"2024-05-19T19:47:48.139681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset\nWe can now write a Dataset object implementing the augmentations and spectrogram computation:","metadata":{}},{"cell_type":"code","source":"# Truncate or pad the audio sample to the desired duration\ndef trunc_or_pad(waveform, audio_len, start_idx='random'):\n    sig_len = waveform.shape[-1]\n    diff_len = abs(sig_len - audio_len)\n\n    if (sig_len > audio_len):\n        # Truncate the signal to the given length\n        if start_idx == 'random':\n            start_idx = np.random.randint(0, diff_len)\n        else:\n            start_idx = 0\n        waveform = waveform[:, start_idx:start_idx + audio_len]\n    \n    elif (sig_len < audio_len):\n        # Length of padding to add at the beginning and end of the signal\n        pad1 = np.random.randint(0, diff_len)\n        pad2 = diff_len - pad1\n        if isinstance(waveform, torch.Tensor):\n            waveform = nn.functional.pad(waveform, pad=(pad1, pad2), mode='constant', value=0)\n        else:\n            waveform = np.pad(waveform, ((0, 0), (pad1, pad2)), mode='constant', constant_values=0)\n    \n    return waveform\n\n\nclass AudioDataset(Dataset):\n    def __init__(\n            self, \n            df, \n            n_classes,\n            start_idx = 'random',\n            duration = 10,\n            sample_rate = 32000,\n            target_length = 384,\n            n_mels = 128,\n            n_fft = 2028,\n            window = 2028,\n            hop_length = None,\n            fmin = 20,\n            fmax = 16000,\n            top_db = 80,\n            waveform_transforms=None,\n            spec_transforms=None,\n            mean=None,\n            std=None,\n            loss='crossentropy',\n            secondary_labels_weight=0.\n            ):\n        super(AudioDataset, self).__init__()\n        self.df = df\n        self.n_classes = n_classes\n        self.start_idx = start_idx\n        self.duration = duration\n        self.sample_rate = sample_rate\n        self.audio_len = duration*sample_rate\n        self.target_length = target_length\n        self.n_mels = n_mels\n        self.n_fft = n_fft\n        self.window = window\n        self.hop_length = self.audio_len // (target_length-1) if not hop_length else hop_length\n        self.fmin = fmin\n        self.fmax = fmax\n        self.top_db = top_db\n        self.loss = loss\n        self.secondary_labels_weight = secondary_labels_weight\n\n        self.to_mel_spectrogramn = nn.Sequential(\n            torchaudio.transforms.MelSpectrogram(self.sample_rate, n_fft=self.n_fft, win_length=self.window,  \n                                                 hop_length=self.hop_length, n_mels=self.n_mels, \n                                                 f_min=self.fmin, f_max=self.fmax),\n            torchaudio.transforms.AmplitudeToDB(top_db=self.top_db)\n        )\n        if mean is not None:\n            self.to_mel_spectrogramn.append(v2.Normalize(mean=mean, std=std))\n\n        self.waveform_transforms = waveform_transforms\n        self.spec_transforms  = spec_transforms\n        \n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        item = self.df.iloc[idx]\n\n        label = torch.tensor(item['target'])\n        if self.loss == 'bce':\n            label = nn.functional.one_hot(label, num_classes=self.n_classes).float()\n            for l in item['secondary_targets']:\n                if l is not None:\n                    label += nn.functional.one_hot(torch.tensor(l), num_classes=self.n_classes)*self.secondary_labels_weight\n\n        file = item['filepath']\n        waveform, sr = torchaudio.load(file)\n        waveform = trunc_or_pad(waveform, self.audio_len, self.start_idx)\n\n        if self.waveform_transforms is not None:\n            waveform = self.waveform_transforms(waveform.squeeze().numpy(), sr)[None, :]\n            waveform = torch.Tensor(waveform)\n\n        spec = self.to_mel_spectrogramn(waveform)\n\n        if self.spec_transforms is not None:\n            spec = self.spec_transforms(spec)\n\n        # expand to 3 channels for imagenet trained models\n        spec = spec.expand(3,-1,-1)\n\n        return spec, label","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:48.142131Z","iopub.execute_input":"2024-05-19T19:47:48.142694Z","iopub.status.idle":"2024-05-19T19:47:48.165598Z","shell.execute_reply.started":"2024-05-19T19:47:48.142659Z","shell.execute_reply":"2024-05-19T19:47:48.164544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = AudioDataset(\n    metadata, \n    n_classes=Config.n_classes,\n    duration=Config.duration,\n    sample_rate=Config.sample_rate,\n    target_length=Config.target_length,\n    n_mels=Config.n_mels,\n    n_fft=Config.n_fft,\n    window=Config.window,\n    hop_length=Config.hop_length,\n    fmin=Config.fmin,\n    fmax=Config.fmax,\n    top_db=Config.top_db,\n    waveform_transforms=waveform_transforms,\n    spec_transforms=spec_transforms,\n    mean=Config.dataset_mean,\n    std=Config.dataset_std,\n    loss=Config.loss,\n    secondary_labels_weight=Config.secondary_labels_weight\n    )","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:48.166897Z","iopub.execute_input":"2024-05-19T19:47:48.1675Z","iopub.status.idle":"2024-05-19T19:47:48.180708Z","shell.execute_reply.started":"2024-05-19T19:47:48.167473Z","shell.execute_reply":"2024-05-19T19:47:48.179866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CutMix / MixUp in Dataloader\nCutMix and MixUp are data augmentation methods which mix two different training samples, both in their data and in their labels. More details in their respective papers: [CutMix](https://arxiv.org/abs/1905.04899) and [MixUp](https://arxiv.org/abs/1710.09412).\n\nAs these transforms use multiple samples, they cannot be implemented in the same way as the previous ones. We can implement them easily with [Torchvision](https://pytorch.org/vision/main/auto_examples/transforms/plot_cutmix_mixup.html) inside of the Dataloader. However, the torchvision transforms expects a number for each label and do not support one-hot encoded labels which you will use with BCE (Binary Cross Entropy) loss. We can easily modify them to add support for one-hot encoded labels. The only modification is in ```_BaseMixUpCutMix``` with ```one_hot_labels```:","metadata":{}},{"cell_type":"code","source":"from typing import Any, Callable, Dict, List, Tuple\nimport math\nimport numbers\nimport warnings\nimport PIL.Image\nfrom torchvision.transforms.v2._utils import _parse_labels_getter, has_any, is_pure_tensor, query_chw, query_size\nfrom torch.nn.functional import one_hot\nfrom torch.utils._pytree import tree_flatten, tree_unflatten\nfrom torchvision import transforms as _transforms, tv_tensors\nfrom torchvision.transforms.v2 import functional as F\nclass _BaseMixUpCutMix(v2.Transform):\n    def __init__(self, *, alpha: float = 1.0, num_classes: int, labels_getter=\"default\", one_hot_labels: bool = False) -> None:\n        super().__init__()\n        self.alpha = float(alpha)\n        self._dist = torch.distributions.Beta(torch.tensor([alpha]), torch.tensor([alpha]))\n\n        self.num_classes = num_classes\n\n        self._labels_getter = _parse_labels_getter(labels_getter)\n        self.one_hot_labels = one_hot_labels\n\n    def forward(self, *inputs):\n        inputs = inputs if len(inputs) > 1 else inputs[0]\n        flat_inputs, spec = tree_flatten(inputs)\n        needs_transform_list = self._needs_transform_list(flat_inputs)\n\n        if has_any(flat_inputs, PIL.Image.Image, tv_tensors.BoundingBoxes, tv_tensors.Mask):\n            raise ValueError(f\"{type(self).__name__}() does not support PIL images, bounding boxes and masks.\")\n\n        labels = self._labels_getter(inputs)\n        if not isinstance(labels, torch.Tensor):\n            raise ValueError(f\"The labels must be a tensor, but got {type(labels)} instead.\")\n        elif labels.ndim != 1 and not self.one_hot_labels:\n            raise ValueError(\n                f\"labels tensor should be of shape (batch_size,) \" f\"but got shape {labels.shape} instead.\"\n            )\n        elif labels.ndim != 2 and labels.size(-1) != self.num_classes and self.one_hot_labels:\n            raise ValueError(\n                f\"labels tensor should be of shape (batch_size, {self.num_classes}) \" f\"but got shape {labels.shape} instead.\"\n            )\n\n        params = {\n            \"labels\": labels,\n            \"batch_size\": labels.shape[0],\n            **self._get_params(\n                [inpt for (inpt, needs_transform) in zip(flat_inputs, needs_transform_list) if needs_transform]\n            ),\n        }\n\n        # By default, the labels will be False inside needs_transform_list, since they are a torch.Tensor coming\n        # after an image or video. However, we need to handle them in _transform, so we make sure to set them to True\n        needs_transform_list[next(idx for idx, inpt in enumerate(flat_inputs) if inpt is labels)] = True\n        flat_outputs = [\n            self._transform(inpt, params) if needs_transform else inpt\n            for (inpt, needs_transform) in zip(flat_inputs, needs_transform_list)\n        ]\n\n        return tree_unflatten(flat_outputs, spec)\n\n    def _check_image_or_video(self, inpt: torch.Tensor, *, batch_size: int):\n        expected_num_dims = 5 if isinstance(inpt, tv_tensors.Video) else 4\n        if inpt.ndim != expected_num_dims:\n            raise ValueError(\n                f\"Expected a batched input with {expected_num_dims} dims, but got {inpt.ndim} dimensions instead.\"\n            )\n        if inpt.shape[0] != batch_size:\n            raise ValueError(\n                f\"The batch size of the image or video does not match the batch size of the labels: \"\n                f\"{inpt.shape[0]} != {batch_size}.\"\n            )\n\n    def _mixup_label(self, label: torch.Tensor, *, lam: float) -> torch.Tensor:\n        if not self.one_hot_labels:\n            label = one_hot(label, num_classes=self.num_classes)\n        if not label.dtype.is_floating_point:\n            label = label.float()\n        return label.roll(1, 0).mul_(1.0 - lam).add_(label.mul(lam))\n\n\nclass MixUp(_BaseMixUpCutMix):\n    \"\"\"Apply MixUp to the provided batch of images and labels.\n\n    Paper: `mixup: Beyond Empirical Risk Minimization <https://arxiv.org/abs/1710.09412>`_.\n\n    .. note::\n        This transform is meant to be used on **batches** of samples, not\n        individual images. See\n        :ref:`sphx_glr_auto_examples_transforms_plot_cutmix_mixup.py` for detailed usage\n        examples.\n        The sample pairing is deterministic and done by matching consecutive\n        samples in the batch, so the batch needs to be shuffled (this is an\n        implementation detail, not a guaranteed convention.)\n\n    In the input, the labels are expected to be a tensor of shape ``(batch_size,)``. They will be transformed\n    into a tensor of shape ``(batch_size, num_classes)``.\n\n    Args:\n        alpha (float, optional): hyperparameter of the Beta distribution used for mixup. Default is 1.\n        num_classes (int): number of classes in the batch. Used for one-hot-encoding.\n        labels_getter (callable or \"default\", optional): indicates how to identify the labels in the input.\n            By default, this will pick the second parameter as the labels if it's a tensor. This covers the most\n            common scenario where this transform is called as ``MixUp()(imgs_batch, labels_batch)``.\n            It can also be a callable that takes the same input as the transform, and returns the labels.\n    \"\"\"\n\n    def _get_params(self, flat_inputs: List[Any]) -> Dict[str, Any]:\n        return dict(lam=float(self._dist.sample(())))  # type: ignore[arg-type]\n\n    def _transform(self, inpt: Any, params: Dict[str, Any]) -> Any:\n        lam = params[\"lam\"]\n\n        if inpt is params[\"labels\"]:\n            return self._mixup_label(inpt, lam=lam)\n        elif isinstance(inpt, (tv_tensors.Image, tv_tensors.Video)) or is_pure_tensor(inpt):\n            self._check_image_or_video(inpt, batch_size=params[\"batch_size\"])\n\n            output = inpt.roll(1, 0).mul_(1.0 - lam).add_(inpt.mul(lam))\n\n            if isinstance(inpt, (tv_tensors.Image, tv_tensors.Video)):\n                output = tv_tensors.wrap(output, like=inpt)\n\n            return output\n        else:\n            return inpt\n\n\nclass CutMix(_BaseMixUpCutMix):\n    \"\"\"Apply CutMix to the provided batch of images and labels.\n\n    Paper: `CutMix: Regularization Strategy to Train Strong Classifiers with Localizable Features\n    <https://arxiv.org/abs/1905.04899>`_.\n\n    .. note::\n        This transform is meant to be used on **batches** of samples, not\n        individual images. See\n        :ref:`sphx_glr_auto_examples_transforms_plot_cutmix_mixup.py` for detailed usage\n        examples.\n        The sample pairing is deterministic and done by matching consecutive\n        samples in the batch, so the batch needs to be shuffled (this is an\n        implementation detail, not a guaranteed convention.)\n\n    In the input, the labels are expected to be a tensor of shape ``(batch_size,)``. They will be transformed\n    into a tensor of shape ``(batch_size, num_classes)``.\n\n    Args:\n        alpha (float, optional): hyperparameter of the Beta distribution used for mixup. Default is 1.\n        num_classes (int): number of classes in the batch. Used for one-hot-encoding.\n        labels_getter (callable or \"default\", optional): indicates how to identify the labels in the input.\n            By default, this will pick the second parameter as the labels if it's a tensor. This covers the most\n            common scenario where this transform is called as ``CutMix()(imgs_batch, labels_batch)``.\n            It can also be a callable that takes the same input as the transform, and returns the labels.\n    \"\"\"\n\n    def _get_params(self, flat_inputs: List[Any]) -> Dict[str, Any]:\n        lam = float(self._dist.sample(()))  # type: ignore[arg-type]\n\n        H, W = query_size(flat_inputs)\n\n        r_x = torch.randint(W, size=(1,))\n        r_y = torch.randint(H, size=(1,))\n\n        r = 0.5 * math.sqrt(1.0 - lam)\n        r_w_half = int(r * W)\n        r_h_half = int(r * H)\n\n        x1 = int(torch.clamp(r_x - r_w_half, min=0))\n        y1 = int(torch.clamp(r_y - r_h_half, min=0))\n        x2 = int(torch.clamp(r_x + r_w_half, max=W))\n        y2 = int(torch.clamp(r_y + r_h_half, max=H))\n        box = (x1, y1, x2, y2)\n\n        lam_adjusted = float(1.0 - (x2 - x1) * (y2 - y1) / (W * H))\n\n        return dict(box=box, lam_adjusted=lam_adjusted)\n\n    def _transform(self, inpt: Any, params: Dict[str, Any]) -> Any:\n        if inpt is params[\"labels\"]:\n            return self._mixup_label(inpt, lam=params[\"lam_adjusted\"])\n        elif isinstance(inpt, (tv_tensors.Image, tv_tensors.Video)) or is_pure_tensor(inpt):\n            self._check_image_or_video(inpt, batch_size=params[\"batch_size\"])\n\n            x1, y1, x2, y2 = params[\"box\"]\n            rolled = inpt.roll(1, 0)\n            output = inpt.clone()\n            output[..., y1:y2, x1:x2] = rolled[..., y1:y2, x1:x2]\n\n            if isinstance(inpt, (tv_tensors.Image, tv_tensors.Video)):\n                output = tv_tensors.wrap(output, like=inpt)\n\n            return output\n        else:\n            return inpt","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:48.181953Z","iopub.execute_input":"2024-05-19T19:47:48.18226Z","iopub.status.idle":"2024-05-19T19:47:48.217222Z","shell.execute_reply.started":"2024-05-19T19:47:48.182235Z","shell.execute_reply":"2024-05-19T19:47:48.21627Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"CutMix and MixUp are (optionnaly) used in the collate function of the dataloader, with a ramdom application. ","metadata":{}},{"cell_type":"code","source":"cutmix_or_mixup = v2.RandomApply([\n    v2.RandomChoice([\n        CutMix(num_classes=Config.n_classes, alpha=0.5, one_hot_labels=Config.loss=='bce'),\n        MixUp(num_classes=Config.n_classes, alpha=0.5, one_hot_labels=Config.loss=='bce')\n    ], p=[0.65, 0.35])\n], p=0.7)\n\n\ndef mix_collate_fn(batch):\n    return cutmix_or_mixup(*default_collate(batch))\n\ncollate_fn = mix_collate_fn if Config.cutmix_mixup else None\n\ntrain_loader = DataLoader(train_dataset, batch_size=Config.batch_size, shuffle=True, num_workers=4, collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:48.2183Z","iopub.execute_input":"2024-05-19T19:47:48.218717Z","iopub.status.idle":"2024-05-19T19:47:48.233828Z","shell.execute_reply.started":"2024-05-19T19:47:48.218683Z","shell.execute_reply":"2024-05-19T19:47:48.232981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Result of all augmentations\nAugmentations visualized for a batch:","metadata":{}},{"cell_type":"code","source":"spec_batch, labels = next(iter(train_loader))\nvisualize_spec_batch(spec_batch)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:47:48.234982Z","iopub.execute_input":"2024-05-19T19:47:48.235331Z","iopub.status.idle":"2024-05-19T19:48:02.682794Z","shell.execute_reply.started":"2024-05-19T19:47:48.235298Z","shell.execute_reply":"2024-05-19T19:48:02.681639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations and Spectrograms on GPU\nData loading can be quite long with the computation of spectrograms and the data augmentations, especially since we cannot use a high number of workers in the dataloader on Kaggle. The following code shows how long it takes to go through the whole dataset (around 25 min on Kaggle):","metadata":{}},{"cell_type":"code","source":"# Uncomment to see the duration on CPU (20-30min)\n#for spec_batch, labels in tqdm(train_loader):\n#    pass","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:48:02.684297Z","iopub.execute_input":"2024-05-19T19:48:02.684636Z","iopub.status.idle":"2024-05-19T19:51:56.878224Z","shell.execute_reply.started":"2024-05-19T19:48:02.684605Z","shell.execute_reply":"2024-05-19T19:51:56.876526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can instead perform the augmentations and spectrogram computation on GPU to avoid the dataloading bottleneck. All augmentations are done after data loading, on batches of data. We first define a Dataset which only returns unmodified audio samples, and a Dataloader which contains the same CutMix and MixUp augments, which are done on the original audio signals this time:","metadata":{}},{"cell_type":"code","source":"# Dataset returning truncated audio samples\nclass AudioDataset(Dataset):\n    def __init__(\n            self, \n            df, \n            n_classes,\n            start_idx = 'random',\n            duration = 10,\n            sample_rate = 32000,\n            loss='crossentropy',\n            secondary_labels_weight=0.\n            ):\n        super(AudioDataset, self).__init__()\n        self.df = df\n        self.n_classes = n_classes\n        self.start_idx = start_idx\n        self.duration = duration\n        self.sample_rate = sample_rate\n        self.audio_len = duration*sample_rate\n        self.loss = loss\n        self.secondary_labels_weight = secondary_labels_weight\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        item = self.df.iloc[idx]\n\n        label = torch.tensor(item['target'])\n        if self.loss == 'bce':\n            label = nn.functional.one_hot(label, num_classes=self.n_classes).float()\n            for l in item['secondary_targets']:\n                if l is not None:\n                    label += nn.functional.one_hot(torch.tensor(l), num_classes=self.n_classes)*self.secondary_labels_weight\n\n        file = item['filepath']\n        waveform, sr = torchaudio.load(file)\n        waveform = trunc_or_pad(waveform, self.audio_len, self.start_idx)\n        return waveform, label\n\ntrain_dataset = AudioDataset(\n    metadata, \n    n_classes=Config.n_classes,\n    duration=Config.duration,\n    sample_rate=Config.sample_rate,\n    loss=Config.loss,\n    secondary_labels_weight=Config.secondary_labels_weight\n    )\n\n# CutMix and MixUp in the dataloader, this time directly on audio files\ncutmix_or_mixup = v2.RandomApply([\n    v2.RandomChoice([\n        CutMix(num_classes=Config.n_classes, alpha=0.5, one_hot_labels=Config.loss=='bce'),\n        MixUp(num_classes=Config.n_classes, alpha=0.5, one_hot_labels=Config.loss=='bce')\n    ], p=[0.65, 0.35])\n], p=0.7)\n\ndef mix_collate_fn(batch):\n    return cutmix_or_mixup(*default_collate(batch))\n\ncollate_fn = mix_collate_fn if Config.cutmix_mixup else None\n\ntrain_loader = DataLoader(train_dataset, batch_size=Config.batch_size, shuffle=True, num_workers=4, collate_fn=collate_fn)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:01.068173Z","iopub.execute_input":"2024-05-19T19:52:01.068832Z","iopub.status.idle":"2024-05-19T19:52:01.083774Z","shell.execute_reply.started":"2024-05-19T19:52:01.068798Z","shell.execute_reply":"2024-05-19T19:52:01.082941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get one batch on device\ndevice = torch.device('cuda')\n\nwaveform_batch, labels = next(iter(train_loader))\nwaveform_batch = waveform_batch.to(device)\nlabels = labels.to(device)\nwaveform_batch.shape, labels.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:01.867428Z","iopub.execute_input":"2024-05-19T19:52:01.868366Z","iopub.status.idle":"2024-05-19T19:52:04.223865Z","shell.execute_reply.started":"2024-05-19T19:52:01.868317Z","shell.execute_reply":"2024-05-19T19:52:04.222747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Audio transforms\nSince Audiomentations is made for CPU data augmentations on numpy array, we instead use the [torch-audiomentations](https://github.com/asteroid-team/torch-audiomentations/tree/main) library. The transforms expect batches of torch tensors this time, can be computed on GPU and support  differentiability if you want to use them in the middle on your model. However, not all Audiomentations transforms are available with torch-audiomentations. Each transform has three parameters to control the random application: ```p``` is the probabilty of applying the transform, ```mode``` controls  whether the transformation is individually applied to each sample of the batch (with different random parameters) or uniformly to the whole batch, and ```p_mode``` if the random choice of applying the transform is for each sample or for the whole batch. More details in the documentation; We set them both to ```per_example``` to have the same behavior as before. Missing transforms can be done on CPU before the others.","metadata":{}},{"cell_type":"code","source":"waveform_transforms = torch_audiomentations.Compose([\n    torch_audiomentations.Shift(min_shift=-0.5, max_shift=0.5, p=0.5, mode='per_example', p_mode='per_example'),\n    \n    torch_audiomentations.Gain(min_gain_in_db=-6., max_gain_in_db=6., p=1, mode='per_example', p_mode='per_example'),\n    \n    torch_audiomentations.AddColoredNoise(min_snr_in_db=5., max_snr_in_db=40., min_f_decay=0, max_f_decay=0, \n                                          p=1., mode='per_example'), # Gaussian (white) Noise\n    torch_audiomentations.AddColoredNoise(min_snr_in_db=5., max_snr_in_db=40., min_f_decay=-3.01, max_f_decay=-3.01, \n                                          p=1., mode='per_example'), # Pink Noise\n\n    # AddBackgroundNoise only supports '.wav' files, you can convert the files or try to modify the transform\n    #torch_audiomentations.AddBackgroundNoise(background_paths=Config.short_noises, min_snr_in_db=3., max_snr_in_db=30., p=0.5),\n    #torch_audiomentations.AddBackgroundNoise(background_paths=Config.background_noises, min_snr_in_db=3., max_snr_in_db=30., p=0.5),\n                                   \n    torch_audiomentations.LowPassFilter(min_cutoff_freq=750., max_cutoff_freq=7500., p=0.8, mode='per_example', p_mode='per_example'),\n    torch_audiomentations.PitchShift(sample_rate=Config.sample_rate, min_transpose_semitones=-2.5, max_transpose_semitones=2.5, \n                                     p=0.3, mode='per_example', p_mode='per_example')\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:04.448059Z","iopub.execute_input":"2024-05-19T19:52:04.448466Z","iopub.status.idle":"2024-05-19T19:52:04.471873Z","shell.execute_reply.started":"2024-05-19T19:52:04.448435Z","shell.execute_reply":"2024-05-19T19:52:04.470916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Apply audio transforms to the batch\nwaveform_batch = waveform_transforms(waveform_batch, sr)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:05.605225Z","iopub.execute_input":"2024-05-19T19:52:05.606113Z","iopub.status.idle":"2024-05-19T19:52:08.01034Z","shell.execute_reply.started":"2024-05-19T19:52:05.606078Z","shell.execute_reply":"2024-05-19T19:52:08.009444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Spectrogram transforms\nThe same torchaudio spectrogram transforms and augmentation can be used on batched tensors, on GPU. ","metadata":{}},{"cell_type":"code","source":"to_mel_spectrogramn = nn.Sequential(\n    torchaudio.transforms.MelSpectrogram(Config.sample_rate, n_fft=Config.n_fft, win_length=Config.window,  \n                                         hop_length=Config.hop_length, n_mels=Config.n_mels, \n                                         f_min=Config.fmin, f_max=Config.fmax),\n    torchaudio.transforms.AmplitudeToDB(top_db=Config.top_db),\n    v2.Normalize(mean=Config.dataset_mean, std=Config.dataset_std)\n).to(device)\n\nspec_transforms = nn.Sequential(\n    FrequencyMaskingAug(0.3, 0.1, Config.n_mels, n_masks=3, mask_mode='mean'),\n    TimeMaskingAug(0.3, 0.1, Config.target_length, n_masks=3, mask_mode='mean'),\n).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:08.01212Z","iopub.execute_input":"2024-05-19T19:52:08.012498Z","iopub.status.idle":"2024-05-19T19:52:08.024222Z","shell.execute_reply.started":"2024-05-19T19:52:08.01245Z","shell.execute_reply":"2024-05-19T19:52:08.023419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute spectrogram and apply transforms\nspec_batch = spec_transforms(to_mel_spectrogramn(waveform_batch))\nvisualize_spec_batch(spec_batch)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:08.142207Z","iopub.execute_input":"2024-05-19T19:52:08.143215Z","iopub.status.idle":"2024-05-19T19:52:10.669327Z","shell.execute_reply.started":"2024-05-19T19:52:08.143171Z","shell.execute_reply":"2024-05-19T19:52:10.668321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  GPU results\nWith spectrograms and augmentations of GPU, the time to go through the whole dataset is reduced to less than 10 minuted on Kaggle:","metadata":{}},{"cell_type":"code","source":"# Uncomment to see the duration on GPU (around 8min)\n#for waveform_batch, labels in tqdm(train_loader):\n#    waveform_batch = waveform_batch.to(device)\n#    labels = labels.to(device)\n#    waveform_batch = waveform_transforms(waveform_batch, sr)\n#    spec_batch = spec_transforms(to_mel_spectrogramn(waveform_batch))","metadata":{"execution":{"iopub.status.busy":"2024-05-19T19:52:18.475831Z","iopub.execute_input":"2024-05-19T19:52:18.476228Z","iopub.status.idle":"2024-05-19T20:00:27.771083Z","shell.execute_reply.started":"2024-05-19T19:52:18.476199Z","shell.execute_reply":"2024-05-19T20:00:27.769872Z"},"trusted":true},"execution_count":null,"outputs":[]}]}